'''
执行数据获取
'''
import ast
import json
import math
import os
import random
import shutil
import subprocess
import threading
import time
import winreg
import zipfile
from collections import defaultdict
from datetime import datetime, timedelta

import numpy as np
import pymysql
import requests
from PIL import Image
from PyQt5.QtCore import QThread, pyqtSignal

from playwright.sync_api import sync_playwright
# import pandas as pd
from pandas import read_csv, read_excel, DataFrame, to_datetime, to_numeric, concat, Series
import xlrd
import openpyxl
from Util.DbProcess import auto_data_to_mysql, query_task_status, only_up_task_status
from Util.zconfig import get_setting
# from fontTools.pens.recordingPen import RecordingPen
# from fontTools.ttLib import TTFontplaywright install
# from scipy.spatial.distance import cdist, directed_hausdorff
import chardet
import re
import pytesseract


class WorkerThread(QThread):
    '''
    获取数据类
    '''
    # 线程信号
    upNextDate = pyqtSignal(str)  # 更新下次运行时间
    upLog = pyqtSignal(str)  # 用于更新日志的信号
    finished = pyqtSignal()  # 结束信号
    status_signal = pyqtSignal(str)  # 状态栏信息

    def __init__(self, page, logger, selected_browser, select_ids, min_wait, max_wait, again_flag,login_data):
        super().__init__()
        self.page = page
        self.Debug = get_setting('DEBUG')
        self.logger = logger
        self.selected_browser = selected_browser  # 用户选择使用的浏览器
        self.select_ids = select_ids  # 用户已选择的店铺ids
        self.min_wait = min_wait
        self.max_wait = max_wait
        self.again_flag = again_flag  # 再次下载标识  True是再次下载
        self.login_data = login_data

    # 检测文件编码的函数
    def detect_encoding(self, file):
        raw_data = file.read(10000)  # 读取文件的前10000个字节进行检测
        result = chardet.detect(raw_data)
        file.seek(0)  # 重置文件指针
        return result['encoding']

    def get_file_to_pd(self, download_folder, suggested_filename):
        '''
        根据文件路径读取数据
        :param download_folder: 文件夹路径
        :param suggested_filename: 文件名
        :return: pd格式的待导入数据
        '''
        df = None
        download_path = os.path.join(download_folder, suggested_filename)  # 获取下载文件的路径
        # 验证下载的文件是否存在
        if not os.path.exists(download_path):
            raise FileNotFoundError(f"该文件不存在: {download_path}")

        # 处理下载的文件
        if download_path.endswith('.csv'):
            # 如果是 CSV 文件，先检测编码再读取
            with open(download_path, 'rb') as f:
                encoding = self.detect_encoding(f)
                if '营销概况_全站营销' in download_path:  # 京东的营销概况读取时特殊处理
                    df = read_csv(f, encoding=encoding, skiprows=1, nrows=15)
                elif '营销概况_矩阵分析' in download_path:  # 京东的非营销概况读取时特殊处理
                    df = read_csv(f, encoding=encoding, skiprows=1, nrows=4)  # 跳过第 1 行，读取第 2-5 行
                else:  # 天猫的
                    df = read_csv(f, encoding=encoding)
        elif download_path.endswith('.xlsx'):
            # 如果是 XLSX 文件，直接读取
            df = read_excel(download_path, engine='openpyxl')
            # 删除列名中包含"同比"字样的列
            df = df.loc[:, ~df.columns.str.contains('同比')]
        elif download_path.endswith('.xls'):
            df = read_excel(download_path, engine='xlrd')
        else:
            # 如果是 ZIP 文件，解压并读取其中的 CSV 文件
            with zipfile.ZipFile(download_path, 'r') as z:
                for filename in z.namelist():
                    if filename.endswith('.csv'):
                        with z.open(filename) as csv_file:
                            # 读取压缩包内的文件
                            with csv_file as f:
                                encoding = self.detect_encoding(f)
                                df = read_csv(f, encoding=encoding)  # 使用检测到的编码读取 CSV
                    else:  # 京东的压缩文件内是xlsx
                        with z.open(filename) as excel_file:
                            df = read_excel(excel_file, engine='openpyxl')

        if not df.empty:
            df.fillna('', inplace=True)
        try:
            os.remove(download_path)
        except FileNotFoundError:
            pass
        return df

    def check_all_task(self, data: dict):
        """
        检查字典中每个店铺的任务是否全部为 True，并返回每个店铺的详细统计数据。
        :param data: 字典，格式与输入字典相同
        :return: 字典，包含店铺名称及其对应的任务状态是否全为 True，以及 False 的数量统计
        """
        result = {}
        for store, tasks in data.items():
            # 统计店铺级别的 False 数量
            store_false_count = sum(1 for task in tasks.values() if isinstance(task, bool) and task == False)
            # 统计产品 ID 级别的 False 数量（只统计有 False 的 ID）
            product_false_count = sum(
                1 for task in tasks.values() if
                isinstance(task, dict) and any(sub_task == False for sub_task in task.values())
            )
            # 检查店铺下所有任务（包括子任务）是否全部为 True
            all_true = all(
                task == True if isinstance(task, bool) else all(sub_task == True for sub_task in task.values()) for task
                in tasks.values())
            result[store] = {
                'all_true': all_true,  # 店铺是否全部获取完
                'store_false_count': str(store_false_count),  # 店铺级数据有几个表是false
                'product_false_count': str(product_false_count)  # 店铺内有几个产品ID的数据有false,不需要统计到有几个false
            }
        return result

    def parse_value(self, value):
        value = value.strip()
        if not value:  # 处理空值情况
            return 0
        if value == '-':  # 显式处理 '-' 的情况
            return 0
        if '%' in value:  # 百分比直接返回字符串
            return value
        value = value.replace(',', '')  # 去除逗号
        return float(value) if '.' in value else int(value)

    # def get_sjzlr(self, page):
    #     table = page.locator('div.ant-table-body table')
    #     # 确保表格加载完成
    #     table.wait_for()
    #     # 只获取前7天的表头日期
    #     date_headers = table.locator('thead th:not(:first-child) .dateLabel')
    #     dates = [date_headers.nth(i).inner_text().strip() + " 00:00:00" for i in range(min(7, date_headers.count()))]
    #
    #     data = []
    #     rows = table.locator('tbody tr')
    #     for row_idx in range(rows.count()):
    #         row = rows.nth(row_idx)
    #         # 获取指标名称
    #         metric_name = row.locator('td:first-child .leading.columnTd').inner_text().strip()
    #
    #         # 获取前7个数值
    #         cells = row.locator('td:not(:first-child) .value')
    #         values = [self.parse_value(cells.nth(i).inner_text()) for i in range(min(7, cells.count()))]
    #
    #         if len(values) == len(dates):
    #             data.append({'指标名称': metric_name, **dict(zip(dates, values))})
    #
    #     df = DataFrame(data).set_index('指标名称').T
    #     df.index.name = '日期'
    #     df.insert(0, '类别', '本店')
    #     # **新增列**
    #     new_columns1 = ["支付老买家数", "回访成交", "咨询人数", "内容查看人数", "有过已购状态的客户数", "旺旺满意度"]
    #     new_columns2 = ["48小时揽收及时率", "关联商品数", "总内容供给规模量", "纠纷投诉商责率","商品覆盖率",
    #                     "开播时长(小时)", "询单转化率", "老买家支付金额", "进入公域视频数","总支付子订单数", "停留时长"]
    #     df[new_columns1] = None
    #     if "支付金额" in df.columns:
    #         df["总支付金额"] = df["支付金额"]
    #     df[new_columns2] = None
    #     df = df.reset_index() #去掉索引列
    #     return df
    # -----------------------------  拼多多字体轨迹坐标对比     start 准确度不高,暂不使用-----------------------------#
    # def base_mapping(self,page):
    #     """
    #     提取基础字体文件中的数字字符和它们的坐标信息，并保存到一个JSON文件中。
    #     """
    #     fonts_folder = os.path.join(os.getcwd(), 'download','fonts')
    #     if not os.path.exists(fonts_folder):
    #         os.makedirs(fonts_folder)
    #     base_font_path = os.path.join(fonts_folder, 'base_pdd_font.ttf')
    #     base_font = TTFont(base_font_path)
    #     base_mapping = {
    #         'period': '.','uniEDC6':'8','uniEB8B':'5','uniECA5':'9','uniEE9F':'0','uniE837':'6','uniEE07':'4','uniE45A':'3','uniE583':'2','uniE8CD':'7','uniEB43':'1'
    #     }
    #     # 下载当前网页使用的字体文件
    #     style_tag = page.query_selector("style[data-id^='spider-']")
    #     font_url = style_tag.inner_text()  # 获取 style 标签中的文本内容
    #     start_index = font_url.find("url('") + 5
    #     end_index = font_url.find("')", start_index)
    #     font_url = font_url[start_index:end_index]
    #     font_resp = requests.get(font_url)
    #     curr_font_path = os.path.join(os.getcwd(), 'download', 'fonts', 'spider-font.ttf')
    #     with open(curr_font_path, 'wb') as f:
    #         f.write(font_resp.content)
    #     # 加载新字体
    #     pdd_font = TTFont(curr_font_path)
    #     return base_font,pdd_font,base_mapping
    #
    # # 提取字体中每个字形的坐标点
    # def get_glyph_coordinates(self,font, glyph_name):
    #     glyf_table = font['glyf']
    #     glyph = glyf_table[glyph_name]
    #     if glyph.isComposite():
    #         return None
    #     return glyph.coordinates.array
    #
    # # 提取字体的 (字形名 → 坐标点) 映射
    # def extract_glyph_coordinates(self,font):
    #     glyf_table = font['glyf']
    #     cmap_table = font['cmap'].getcmap(3, 1).cmap
    #     return {glyph_name: self.get_glyph_coordinates(font, glyph_name) for glyph_name in glyf_table.keys() if glyph_name in cmap_table.values()}
    #
    # def normalize_coords(self,coords):
    #     if len(coords) == 0:
    #         return coords
    #     # 计算质心并平移至原点
    #     centroid = np.mean(coords, axis=0)
    #     translated = coords - centroid
    #     # 缩放到单位范围（最大绝对值）
    #     max_val = np.max(np.abs(translated))
    #     if max_val > 0:
    #         translated /= max_val
    #     return translated
    #
    # # 使用 Hausdorff 距离比较字形相似度
    # def compare_glyphs(self,coords1, coords2,threshold=0.18):
    #     if coords1 is None or coords2 is None:
    #         return False
    #     zthreshold = threshold
    #     # 转换为一维数组并重塑为N×2
    #     zcoords1 = np.array(coords1).reshape(-1, 2)
    #     zcoords2 = np.array(coords2).reshape(-1, 2)
    #     zcoords1 = self.normalize_coords(zcoords1)
    #     zcoords2 = self.normalize_coords(zcoords2)
    #
    #     # 计算双向Hausdorff距离
    #     d1 = directed_hausdorff(zcoords1, zcoords2)[0]
    #     d2 = directed_hausdorff(zcoords2, zcoords1)[0]
    #     # if d1 <= 0.114 or d2 <= 0.114:
    #     if ( d1 < 0.114 and d2 < 0.28 ) or (d2 < 0.114 and d1 < 0.28 ) or d1 < 0.1 or d2 < 0.1:#8he3
    #         max_h = min(d1, d2)
    #     else:
    #         if (d1 < 0.199 and d2 < 0.31 ) or (d2 < 0.199 and d1 < 0.31):
    #             if abs(d1 - d2) < 0.02:
    #                 max_h = max(d1, d2)
    #             else:
    #                 max_h = min(d1, d2)
    #                 zthreshold = 0.199
    #         else:
    #             if abs(d1 - d2) < 0.09 and d1 < 0.3 and d2 < 0.3:
    #                 zthreshold = 0.28
    #             if abs(d1 - d2) < 0.009 and d1 < 0.28 and d2 < 0.28:#8,6
    #                 zthreshold = 0.26
    #                 if d1 < 0.22 or d2 < 0.22:
    #                     zthreshold = 0.22
    #             if abs(d1 - d2) < 0.02 and d1 < 0.19 and d2 < 0.19:#6,8
    #                 zthreshold = 0.15
    #
    #             max_h = max(d1, d2)
    #     # 动态调整阈值
    #     similarity = max_h < zthreshold
    #     print(f'正向相似度:{d1}',f'反向相似度:{d2}',f'最终相似度:{max_h}',f'对比结果:{similarity}')
    #     return similarity  # 阈值根据字体情况调整
    #
    # # 构造拼多多 Unicode → 数字 的映射
    # def generate_pdd_mapping(self,base_font, pdd_font, base_mapping):
    #     base_glyphs = self.extract_glyph_coordinates(base_font)
    #     pdd_glyphs = self.extract_glyph_coordinates(pdd_font)
    #     del pdd_glyphs['period']
    #     pdd_to_num_map = {'period':'.'}
    #     matched_pdd_glyphs = {'period'}  # 用于记录已匹配的pdd字形，提升查找效率
    #     for base_glyph_name, base_coords in base_glyphs.items():
    #         if base_glyph_name in matched_pdd_glyphs:
    #             continue
    #         for pdd_glyph_name in list(pdd_glyphs.keys()):  # 转换为列表，支持动态删除
    #             if pdd_glyph_name in matched_pdd_glyphs:
    #                 continue  # 跳过已匹配的pdd字形
    #
    #             if self.compare_glyphs(base_coords, pdd_glyphs[pdd_glyph_name]):
    #                 if base_glyph_name in base_mapping:
    #                     pdd_to_num_map[pdd_glyph_name] = base_mapping[base_glyph_name]
    #                     matched_pdd_glyphs.add(pdd_glyph_name)  # 记录匹配成功的字形
    #                     del pdd_glyphs[pdd_glyph_name]  # 直接从字典中删除已匹配项
    #                     print(f'-----------------------------数字{base_mapping[base_glyph_name]}:{pdd_glyph_name}-----------------------------')
    #                     break  # 跳出当前循环，避免重复匹配
    #             else:
    #                 print(f'-----------------------------数字{base_mapping[base_glyph_name]}对比失败-----------------------------')
    #     return pdd_to_num_map
    #
    # # 替换网页中的加密字符
    # def replace_encrypted_numbers(self,html_content, pdd_to_num_map):
    #     for unicode_char, num in pdd_to_num_map.items():
    #         unicode_code = f'&#x{ord(unicode_char):x};'
    #         html_content = html_content.replace(unicode_code, num)
    #     return html_content
    # -----------------------------  拼多多字体轨迹坐标对比     end-----------------------------#
    #     def recognize_digits_from_image(self,image, min_area=100):
    #         # 转灰度并二值化
    #         gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    #         _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
    #
    #         # 查找轮廓
    #         contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    #
    #         digits = []
    #         for contour in contours:
    #             if cv2.contourArea(contour) > min_area:  # 过滤噪点
    #                 x, y, w, h = cv2.boundingRect(contour)
    #                 roi = thresh[y:y + h, x:x + w]
    #
    #                 # 调整尺寸并归一化
    #                 roi = cv2.resize(roi, (28, 28))
    #                 roi = roi / 255.0
    #
    #                 # 使用 kNN 训练模型识别数字
    #                 digits.append((x, roi))
    #
    #         # 按横坐标排序，确保数字顺序正确
    #         digits.sort(key=lambda x: x[0])
    #
    #         # 加载KNN模型并识别
    #         model = cv2.ml.KNearest_create()
    #         digits_data = np.loadtxt('digits_data.csv', dtype=np.float32)
    #         labels = np.loadtxt('digits_labels.csv', dtype=np.float32)
    #         model.train(digits_data, cv2.ml.ROW_SAMPLE, labels)
    #
    #         result = ''.join(str(int(model.findNearest(np.array([d[1].flatten()], dtype=np.float32), k=1)[1][0][0])) for d in digits)
    #         return result

    def get_ocr_path(self):
        common_paths = [
            r"C:\Program Files\Tesseract-OCR\tesseract.exe",
            r"C:\Program Files (x86)\Tesseract-OCR\tesseract.exe"
        ]
        for path in common_paths:
            if os.path.exists(path):
                return path
        return None

    def ocr_img_num(self, image_path, exe_path):
        pytesseract.pytesseract.tesseract_cmd = exe_path
        image = Image.open(image_path)
        # text = pytesseract.image_to_string(image, config='--psm 6 outputbase digits')#专注于数字
        # OCR 识别（仅允许数字和 .，并去除换行符）
        config = '--psm 6 -c tessedit_char_whitelist=0123456789.,%'
        text = pytesseract.image_to_string(image, config=config).replace("\n", "").strip()
        number = text.replace(",", "").strip()
        return number

    def get_filename(self, filename):
        """清除文件名中的非法字符"""
        return re.sub(r'[\\/*?:"<>|]', "", filename).strip()

    # ===========================广告处理类==================================
    def remove_jd_ad(self, page):
        '''
        京东平台统一处理广告弹窗
        :param page:
        :return:
        '''
        # 移除公告类弹窗
        try:
            page.evaluate("document.querySelectorAll('.bd-notice-modal').forEach(el => el.remove());")
        except BaseException as e:
            print(f"移除广告失败:{str(e)}")
            pass

    def remove_element(self, page, type, element_id):
        '''
        根据页面元素id或类名移除元素
        :param element_id:
        :param type:id或class
        :return:
        '''
        # 检查元素是否存在
        if type == 'id':
            element = page.query_selector(f"#{element_id}")
            if element:
                # 如果元素存在，使用 JavaScript 移除它
                page.evaluate(f"""
                    () => {{
                        const element = document.getElementById('{element_id}');
                        if (element) {{
                            element.remove();
                        }}
                    }}
                """)
            else:
                pass
        elif type == 'class':
            element = page.query_selector(f".{element_id}")
            if element:
                # 如果元素存在，使用 JavaScript 移除它
                page.evaluate(f"""
                        () => {{
                            const elements = document.querySelectorAll('.{element_id}');
                            elements.forEach(element => element.remove());
                        }}
                    """)
            else:
                pass
        else:
            print(f"'{type}' 类型的页面元素不能处理")

    def fill_value(self, page, selector: str, text: str, min_delay=0.1, max_delay=0.9, clear_first=True):
        """
        模拟人类输入
        参数:
            page: Playwright 的 page 实例
            selector: CSS 选择器（如 '#usernameId'）
            text: 要输入的字符串
            min_delay: 单字符输入最小间隔（秒）
            max_delay: 单字符输入最大间隔（秒）
            clear_first: 是否先清空原始内容
        """
        locator = page.locator(selector)
        locator.click()
        if clear_first:
            locator.fill('')
        for char in text:
            locator.type(char, delay=0)
            time.sleep(random.uniform(min_delay, max_delay))

    def run(self):
        def on_page_opened(page):
            global current_page
            current_page = page
            print("New page opened:", current_page.url)

        page = None
        context = None
        base_dir = os.getcwd()
        download_folder = os.path.join(base_dir, 'download')
        if not os.path.exists(download_folder):
            os.makedirs(download_folder)
        auto_cookie_folder = os.path.join(base_dir, 'cookies/')

        now = datetime.now()
        yesterdayTime = now - timedelta(days=1)
        yesterdayTime7 = now - timedelta(days=7)
        yesterdayTime15 = now - timedelta(days=15)
        today = now.strftime('%Y-%m-%d')  # 今天的日期,也用于记录今日的下载任务状态,重要!2025-01-23
        today_23 = now.strftime('%Y-%m-%d %H:00:00')
        today_23_st = now.strftime('%Y-%m-%d %H') + '%'
        # today_23 = now.strftime('2024-12-27 23:00:00')
        # today_23_st = now.strftime('2024-12-27 23') + '%'
        yesterday = yesterdayTime.strftime('%Y-%m-%d')  # 昨天的日期
        yesterday7 = yesterdayTime7.strftime('%Y-%m-%d')  # 7天的日期
        yesterday15 = yesterdayTime15.strftime('%Y-%m-%d')  # 15天的日期
        yesterday0 = yesterdayTime.strftime('%Y-%m-%d 00:00:00')  # 昨天的0点
        yesterday07 = yesterdayTime7.strftime('%Y-%m-%d 00:00:00')  # 7天的0点
        yesterday015 = yesterdayTime15.strftime('%Y-%m-%d 00:00:00')  # 15天的0点
        if self.Debug:
            dburl = get_setting('DBURL')['DEV']
        else:
            dburl = get_setting('DBURL')['PRD']
        # ==============================   检查任务主数据 =====================================================
        try:
            connection = pymysql.connect(
                host=dburl,
                user='root',  # 数据库用户名
                password='123',  # 数据库密码
                database='ecadb'  # 数据库名称
            )
            cursor = connection.cursor()

            # 执行 SQL 查询
            sql_query = """
                SELECT name,id ,store_type
                FROM ecapp_storename 
                WHERE  auto_fetch_data = TRUE and id in %s
                """
            # WHERE store_type = %s AND auto_fetch_data = TRUE
            # cursor.execute(sql_query, ('天猫',))  # 使用元组传递参数  改成取全部平台的店铺的
            cursor.execute(sql_query, (self.select_ids,))  # 使用元组传递参数

            # 获取查询结果
            store_names = cursor.fetchall()

            # 将查询结果格式化为列表
            store_name_list = [row[0] for row in store_names]
            store_id_list = [row[1] for row in store_names]
            store_type = {row[0]: row[2] for row in store_names}
            params = ', '.join(['%s'] * len(store_name_list))
            sql_query = f"""
                        SELECT product_id, name 
                        FROM ecapp_prostoremapping 
                        WHERE name IN ({params}) and auto_fetch_data = True
                        """
            cursor.execute(sql_query, store_name_list)
            results = cursor.fetchall()
            field_names = [desc[0] for desc in cursor.description]  # 获取字段名
            product_dict = [dict(zip(field_names, row)) for row in results]  # 转换为字典列表

            sql_query = f"""
                            SELECT product_id, name 
                            FROM ecapp_prostoremapping 
                            WHERE name IN %s and auto_fetch_data = True
                        """
            cursor.execute(sql_query, (['白雪文具官方旗舰店'],))
            results = cursor.fetchall()
            field_names = [desc[0] for desc in cursor.description]  # 获取字段名
            product_jd_dict = [dict(zip(field_names, row)) for row in results]

            sql_query = """
                           SELECT `key`, `value`
                           FROM ecapp_siteconfig 
                           WHERE `key` = 'JDStoreCategory'
                        """
            cursor.execute(sql_query, )
            results = cursor.fetchall()
            if results:
                key, value_str = results[0]  # 取第一行数据
                jdstore_dict = ast.literal_eval(value_str)  # 转换字符串为字典
            else:
                jdstore_dict = {}

            # 查询今天有没有获取昨天的计划和营销场景数据
            query = f"""
                    SELECT DISTINCT name
                    FROM ecapp_jh
                    WHERE date(date) = %s
                    AND name IN ({params})
                    AND cost > 0
                    ORDER BY name;
                    """
            # 执行查询
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_jh = cursor.fetchall()
            result_jh = {entry[0]: True for entry in queryset_jh}

            # 如果 name 不在查询结果中，设为 False
            for name in store_name_list:
                if name not in result_jh:
                    result_jh[name] = False

            query = f"""
                    SELECT DISTINCT name
                    FROM ecapp_yxcj
                    WHERE date(date) = %s
                    AND name IN ({params})
                    AND cost > 0
                    ORDER BY name;
                    """
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_yxcj = cursor.fetchall()

            # 构建返回的字典
            result_yxcj = {entry[0]: True for entry in queryset_yxcj}

            # 如果 name 不在查询结果中，设为 False
            for name in store_name_list:
                if name not in result_yxcj:
                    result_yxcj[name] = False

            # query = f"""SELECT DISTINCT name FROM ecapp_SJZL
            #         WHERE date = %s AND category = '今天' AND  page_views > 0 AND name IN ({params}) ; """
            # cursor.execute(query, (today_23, *store_name_list))
            # queryset_SJZL = cursor.fetchall()
            # result_SJZL = [row[0] for row in queryset_SJZL]
            result_SJZL = []  # 数据数据总览-实时不再使用  2025-01-16
            # today = '2024-12-16'
            query = f"""SELECT DISTINCT product_id FROM ecapp_hxgk
                    WHERE date(date) = %s AND name IN ({params}) ; """
            cursor.execute(query, (today, *store_name_list))
            queryset_hxgk = cursor.fetchall()
            # 当前的整点的数据已下载完的商品ID
            result_hxgk = [row[0] for row in queryset_hxgk]

            # query = f"""SELECT DISTINCT product_id FROM ecapp_LLLY
            #             WHERE date = %s AND name IN ({params}) ; """
            # cursor.execute(query, (today_23, *store_name_list))
            # queryset_LLLY = cursor.fetchall()
            # result_LLLY = [row[0] for row in queryset_LLLY]
            result_LLLY = []  # 流量来源-实时不再使用  2025-01-16

            # query = f"""SELECT DISTINCT product_id FROM ecapp_LLLYST
            #             WHERE date LIKE %s AND name IN ({params}) ; """
            # cursor.execute(query, (today_23_st, *store_name_list))
            # queryset_LLLYST = cursor.fetchall()
            # result_LLLYST = [row[0] for row in queryset_LLLYST]
            result_LLLYST = []  # 流量来源手淘-实时不再使用  2025-01-16

            # ================================= 日数据检查================================
            query = f"""SELECT DISTINCT name FROM ecapp_SJZLR
                        WHERE date(date) = %s AND  page_views > 0 AND name IN ({params}) ; """
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_SJZLR = cursor.fetchall()
            result_SJZLR = [row[0] for row in queryset_SJZLR]

            query = f"""SELECT DISTINCT name FROM ecapp_LLLYDPR
                    WHERE date(date) = %s AND name IN ({params}) ; """
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_LLLYDPR = cursor.fetchall()
            result_LLLYDPR = [row[0] for row in queryset_LLLYDPR]

            query = f"""SELECT product_id FROM ecapp_hxgkr
                    WHERE date(date) = %s AND name IN ({params}) and page_views > 0; """
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_hxgkr = cursor.fetchall()
            # 当前的整点的数据已下载完的商品ID
            result_hxgkr = [row[0] for row in queryset_hxgkr]

            query = f"""SELECT DISTINCT product_id FROM ecapp_LLLYR
                        WHERE date(date) = %s AND name IN ({params}) ; """
            cursor.execute(query, (yesterday, *store_name_list))
            queryset_LLLYR = cursor.fetchall()
            result_LLLYR = [row[0] for row in queryset_LLLYR]
            # ================================= 生成初始任务状态信息================================
            # 初始化结果字典
            task_status = defaultdict(dict)
            # 定义默认任务状态
            # store_tasks = ['YXCJ', 'JH', 'SJZLR', 'LLLYDPR', 'ZTR_GJ', 'ZTR_RQ', 'ZTR_HP', 'ZTR_QZ']
            # product_tasks = ['HXGKR', 'LLLYR', 'HXGK']  新版本不再下载HXGK的昨日实时数据
            store_tasks = ['YXCJ', 'SJZLR', 'LLLYDPR', 'ZTR_GJ', 'ZTR_RQ', 'ZTR_HP', 'ZTR_QZ']
            product_tasks = ['HXGKR', 'LLLYR']

            query = """
                  SELECT store_type, name, auto_fetch_data FROM ecapp_storename
                """
            cursor.execute(query)
            results = cursor.fetchall()
            # 1.先生成所有店铺级别的
            for storeType, store_name, auto in results:
                if store_name not in task_status:
                    task_status[store_name] = {task: False for task in store_tasks}

            query = """
                        SELECT product_id, name,auto_fetch_data FROM ecapp_ProStoreMapping
                    """
            cursor.execute(query)
            results = cursor.fetchall()  # 获取查询结果
            # 遍历查询结果，填充任务状态
            for product_id, store_name, auto in results:
                # 确保店铺任务状态存在
                if store_name not in task_status:
                    task_status[store_name] = {task: False for task in store_tasks}
                # 添加产品任务状态到对应店铺
                task_status[store_name][product_id] = {task: False for task in product_tasks}

            task_status = dict(task_status)
            # 2.获取数据库中的任务状态
            query_status = query_task_status(cursor, task_date=today)
            # 3.根据数据库中的值更新状态
            if query_status:
                for store, store_status in query_status.items():
                    if store in task_status:
                        # 更新店铺状态
                        for key, status in store_status.items():
                            if isinstance(status, dict):  # 如果是商品 ID 的字典
                                # 如果该商品 ID 在 initial_status 里有对应项，则更新
                                for table, product_table_status in status.items():
                                    if key in task_status[store]:
                                        task_status[store][key][table] = product_table_status
                            else:
                                # 否则直接更新店铺层级的状态
                                task_status[store][key] = status
            if self.again_flag:  # 如果勾选再次下载,则把选中的店铺的任务状态重置为False  LJ 2025-03-27
                for store_name in store_name_list:
                    if store_name in task_status:
                        # 将店铺任务状态设置为 False
                        for task in store_tasks:
                            task_status[store_name][task] = False
                        # 将产品任务状态设置为 False
                        for product_id, product_tasks in task_status[store_name].items():
                            if isinstance(product_tasks, dict):  # 确保是产品任务字典
                                for task in product_tasks:
                                    task_status[store_name][product_id][task] = False
            check_task = self.check_all_task(task_status)

            self.logger.info('已获取任务主数据')
        except BaseException as e:
            store_name_list = []
            result_yxcj = {}
            result_jh = {}
            proMsg = '获取任务主数据失败:' + str(e)
            self.logger.error(proMsg)
            raise
        try:
            playwright = sync_playwright().start()
        except BaseException as e:
            self.logger.error('自动化实例创建失败')
            raise
        if self.selected_browser == 'msedge':
            try:
                # 打开注册表键，获取 Microsoft Edge 的路径
                with winreg.OpenKey(winreg.HKEY_CURRENT_USER,
                                    r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\msedge.exe') as key:
                    chrome_path, _ = winreg.QueryValueEx(key, None)
            except FileNotFoundError:
                try:
                    with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
                                        r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\msedge.exe') as key:
                        chrome_path, _ = winreg.QueryValueEx(key, None)
                except FileNotFoundError:
                    self.logger.info('未找到 Microsoft Edge')
                    raise
        else:
            try:
                # 打开注册表键,先找当前用户的注册表,没有再找全局注册表
                with winreg.OpenKey(winreg.HKEY_CURRENT_USER,
                                    r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe') as key:
                    chrome_path, _ = winreg.QueryValueEx(key, None)
            except FileNotFoundError:
                try:
                    with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,
                                        r'SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\chrome.exe') as key:
                        chrome_path, _ = winreg.QueryValueEx(key, None)
                except FileNotFoundError:
                    self.logger.info('未找到谷歌浏览器')
                    raise

        path_user_data = r'C:/new/AutomationProfile'
        path_to_extension = r'C:/new/AutomationProfile/AutoGET/店透视-Chrome_361'
        try:
            browser = playwright.chromium.connect_over_cdp(get_setting('CDPURL'))
            browser_flag = 'Manual'  # 手动
        except BaseException as e:
            browser_flag = 'Auto'  # 自动
        # =========================      日维度数据获取      =========================#
        try:
            yxcj_file_name_value = ''
            jh_file_name_value = ''
            # zindex = 0
            img_path = os.path.join(os.getcwd(), get_setting('TESSERACT')['TESSERACT_IMG'])
            exe_path = self.get_ocr_path()
            os.makedirs(img_path, exist_ok=True)
            for index, store in enumerate(store_name_list):
                try:
                    store_category = jdstore_dict[store]  # 京东使用,区分自营和POP
                except:
                    store_category = '自营'
                if store not in check_task:
                    proMsg = '--' + store + '表不存在于[商品配置表]中,已跳过该店铺'
                    self.logger.warning(proMsg)
                    continue
                if not check_task[store]['all_true']:  # 本店铺未全部完成
                    store_flag = False  # 用来判断是否下载过店铺的任何数据,如果下载过则=True下方的店铺循环的每次循环增加随机等待时间55-120秒
                    pro_flag = False  # 用来判断是否下载过产品的任何数据,如果下载过则=True下方的产品循环的每次循环增加随机等待1-5秒
                    cookies_name = auto_cookie_folder + 'tb/' + str(store_id_list[index]) + '.json'
                    if browser_flag == 'Manual':
                        if os.path.exists(cookies_name):
                            try:
                                with open(cookies_name, 'r') as file:
                                    cookies_data = json.load(file)['cookies']
                            except BaseException as e:
                                self.logger.error(f'登录数据缓存文件{cookies_name}加载失败:{str(e)}')
                                raise
                        else:
                            self.logger.info(
                                f'未找到店铺:{store}的缓存文件:{cookies_name},请登录网站后点击[5.保存登录信息]按钮!')
                            raise
                        context = browser.contexts[0]
                        page = context.pages[0]
                        page.context.add_cookies(cookies_data)  # 使用 page.context 添加 cookies
                        page.reload()  # 重新加载页面，使 cookies 生效
                    else:
                        path_user_data = os.path.join(os.getcwd(), 'cookies', 'AutomationProfile', store)
                        shutil.rmtree(path_user_data, ignore_errors=True)
                        result = next((d for d in self.login_data if d['store_name'] == store), None)
                        if result is not None:
                            try:
                                response = requests.get(get_setting('CDPURL'))
                                if response.status_code == 200:
                                    browser = playwright.chromium.connect_over_cdp(get_setting('CDPURL'))
                            except requests.exceptions.RequestException:
                                # 启动浏览器
                                command = f'"{chrome_path}" --remote-debugging-port=9333 --user-data-dir="{path_user_data}" --disable-blink-features=AutomationControlled --disable-infobars  --no-first-run --no-default-browser-check'
                                subprocess.Popen(command, shell=True)
                                for _ in range(20):
                                    try:
                                        response = requests.get(get_setting('CDPURL'))
                                        if response.status_code == 200:
                                            msg = '指定端口谷歌浏览器已启动，请登录或点击开始定时任务'
                                            browser = playwright.chromium.connect_over_cdp(get_setting('CDPURL'))
                                            break
                                    except Exception as e:
                                        pass
                                    time.sleep(0.5)
                            context = browser.contexts[0]
                            page = context.pages[0]
                            login_url = get_setting('TASK_URL')['PDDLOGIN']
                            page.goto(login_url)
                            page.wait_for_load_state()
                            #拼多多登录用
                            login_div = page.locator("div.login-tab > div.tab-operate").first
                            login_title = login_div.locator("div:has-text('账号登录')").first
                            if login_title.count() > 0:
                                login_title.click(timeout=3000)
                                page.wait_for_load_state()
                            else:
                                logout_btn = page.query_selector("div.logout")
                                if logout_btn:
                                    desc = logout_btn.query_selector("div.logoutDesc")
                                    if desc and "退出当前账号" in desc.inner_text():
                                        logout_btn.click(force=True)
                                        page.wait_for_load_state()
                            if result['username'] and result['password']:
                                self.fill_value(page, '#usernameId', result['username'])
                                self.fill_value(page, '#passwordId', result['password'])
                                loginBtn = page.locator('button:has-text("登录")')
                                loginBtn.wait_for(state="visible", timeout=5000)
                                loginBtn.click(timeout=5000)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                #增加如显示需要填验证码,则等待20秒验证码输入
                                locator = page.locator('input[placeholder="请输入短信验证码"]')
                                if locator.is_visible(timeout=5000):  # 最多等待3秒检测是否出现
                                    self.logger.info("检测到验证码输入框，等待 60 秒")
                                    max_wait = 60  # 最长等待秒数
                                    interval = 2  # 每次检查间隔秒数
                                    waited = 0

                                    while waited < max_wait:
                                        try:
                                            title = page.title()
                                            if "首页" in title:
                                                self.logger.info("检测已登录到店铺，继续执行")
                                                break
                                        except Exception as e:
                                            pass
                                        waited += interval
                                        time.sleep(interval)
                                        zm = max_wait-waited
                                        self.logger.info(f"等待输入验证码: {str(zm)}秒")
                                    else:
                                        raise ValueError("等待超时，用户未登录到店铺")
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                            else:
                                self.logger.info("等待用户手动登录店铺(最长等待时间:30秒)")
                                max_wait = 30  # 最长等待秒数
                                interval = 3  # 每次检查间隔秒数
                                waited = 0

                                while waited < max_wait:
                                    try:
                                        title = page.title()
                                        if "首页" in title:
                                            self.logger.info("检测已登录到店铺，继续执行")
                                            break
                                    except Exception as e:
                                        pass
                                    time.sleep(interval)
                                    waited += interval
                                else:
                                    raise ValueError("等待超时，用户未登录到店铺")
                        else:
                            # 检查文件是否存在
                            if os.path.exists(cookies_name):
                                try:
                                    with open(cookies_name, 'r') as file:
                                        cookies_data = json.load(file)['cookies']
                                    user_agents = [
                                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
                                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
                                        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/89.0 Safari/537.36"
                                    ]
                                    languages = ['zh-CN', 'en-US'],
                                    timezones = ['Asia/Shanghai']  # 'America/New_York', 'Europe/London', 'Australia/Sydney'
                                    # 随机选择一个指纹
                                    user_agent = random.choice(user_agents)
                                    language = random.choice(languages)
                                    timezone = random.choice(timezones)
                                    # path_user_data = os.path.join(os.getcwd(), 'cookies', 'AutomationProfile', store)
                                    # shutil.rmtree(path_user_data, ignore_errors=True)

                                    browser = playwright.chromium.launch_persistent_context(user_data_dir=path_user_data,
                                                                                            executable_path=chrome_path,
                                                                                            # 要想通过这个下载文件这个必然要开  默认是False
                                                                                            accept_downloads=True,
                                                                                            # 设置不是无头模式
                                                                                            headless=False,
                                                                                            bypass_csp=True,
                                                                                            slow_mo=10,
                                                                                            channel=f"{self.selected_browser}",
                                                                                            args=[
                                                                                                '--disable-blink-features=AutomationControlled',
                                                                                                '--remote-debugging-port=9333',
                                                                                                # f"--disable-extensions-except={path_to_extension}",
                                                                                                # f"--load-extension={path_to_extension}",
                                                                                                f'--user-agent={user_agent}',
                                                                                                f'--lang={language}',
                                                                                                # 设置随机语言
                                                                                                f'--timezone={timezone}',
                                                                                                # 设置随机时区
                                                                                            ])
                                    browser.clear_cookies()
                                    # 添加 cookies
                                    browser.add_cookies(cookies_data)
                                    page = browser.pages[0]

                                    # 防止被检测为自动化行为
                                    page.evaluate("""
                                        Object.defineProperty(navigator, 'webdriver', {get: () => undefined})
                                    """)
                                except BaseException as e:
                                    self.logger.error(f'登录数据缓存文件{cookies_name}加载失败:{str(e)}')
                                    raise
                            else:
                                self.logger.info(f'未找到店铺:{store}的缓存文件:{cookies_name},请登录网站后点击[5.保存登录信息]按钮!')
                                raise
                    # 获取该店铺内有多少自动获取数据的商品id
                    product_list = []
                    product_list = [item['product_id'] for item in product_dict if item['name'] == store]
                    product_jd_list = [item['product_id'] for item in product_jd_dict if item['name'] == '白雪文具官方旗舰店']
                    product_list_bx = product_list + product_jd_list
                    proMsg = '--' + store + '--当前状态:[' + check_task[store]['store_false_count'] + ']个店铺级数据表未获取完成' + ',[' + check_task[store]['product_false_count'] + ']个产品未获取完成'
                    self.logger.info(proMsg)
                    if store_type[store] == '天猫':
                        proMsg = '--' + store + '开始检查/获取店铺级数据'
                        self.logger.info(proMsg)
                        # =========================      1.营销场景-日      =========================#
                        self.status_signal.emit("下载营销场景...")
                        if not task_status[store]['YXCJ']:
                            try:
                                yxcj_url = get_setting('TASK_URL')[
                                               'YXCJ'] + yesterday7 + '&endTime=' + yesterday + '&splitType=day&queryDomains=%5B"date"%2C"scene"%5D'
                                page.goto(yxcj_url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 检查登录数据是否过期
                                login_btn = page.locator('//button[span[text()="立即登录"]]')
                                if login_btn.count() > 0:
                                    proMsg = '--' + store + '登录数据已过期,请重新保存登录数据后再重新下载本店铺数据!'
                                    self.logger.error(proMsg)
                                    try:
                                        context.close()
                                    except:
                                        pass
                                    try:
                                        browser.close()
                                    except:
                                        pass
                                    continue

                                while True:
                                    try:
                                        button_locator = page.locator('button:has-text("下载报表")')
                                        button_locator.scroll_into_view_if_needed()
                                        button_locator.click()
                                        break
                                    except:
                                        time.sleep(random.randint(self.min_wait, self.max_wait))
                                        page.goto(yxcj_url)
                                        page.wait_for_load_state()
                                        button_locator = page.locator('button:has-text("下载报表")')
                                        button_locator.scroll_into_view_if_needed()
                                        button_locator.click()
                                        time.sleep(random.randint(self.min_wait, self.max_wait))

                                div_dialog_body = page.locator('div.dialog-body').last
                                # 获取该元素的父级 div
                                div_dialog = div_dialog_body.locator('..')
                                file_name_input = div_dialog.locator('input[value^="营销场景报表_"]')
                                yxcj_file_name_value = file_name_input.input_value()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 下载按钮
                                div_dialog.locator('button:has-text("确定")').click()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                store_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--营销场景数据获取失败:' + str(e)
                                self.logger.error(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # # =========================      2.计划报表-日  不再下载计划表 2025-03-24      =========================#
                        # self.status_signal.emit("下载计划报表...")
                        if 1 == 2 and not task_status[store]['JH']:
                            try:
                                jh_url = get_setting('TASK_URL')[
                                             'JH'] + yesterday + '&endTime=' + yesterday + '&splitType=hour'
                                page.goto(jh_url)
                                page.wait_for_load_state()
                                button_locator = page.locator('button:has-text("下载报表")')
                                button_locator.scroll_into_view_if_needed()
                                button_locator.click()
                                # while True:
                                #     try:
                                #         page.wait_for_selector('.dialog-header-name', timeout=5000)
                                #         break
                                #     except BaseException as e:
                                #         button_locator.click()
                                #         pass
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                # jh_div_dialog_header = page.locator('div[data-spm="onebp_views_pages_report_download-dialog"] div.dialog-header:has(span:has-text("下载报表"))')
                                jh_div_dialog_body = page.locator('div.dialog-body').last
                                # 获取该元素的父级 div
                                jh_div_dialog = jh_div_dialog_body.locator('..')
                                jh_file_name_input = jh_div_dialog.locator('input[value^="计划报表_"]')
                                jh_file_name_value = jh_file_name_input.input_value()
                                # 下载按钮
                                jh_div_dialog.locator('button:has-text("确定")').click()
                                time.sleep(random.randint(self.min_wait, self.max_wait))

                                store_flag = True
                            except BaseException as e:
                                jh_file_name_value = ''
                                proMsg = '--' + store + '--计划报表获取失败:' + str(e)
                                self.logger.error(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      获取下载内容      =========================#
                        self.status_signal.emit("获取营销场景和计划数据...")
                        if not task_status[store]['YXCJ']:
                            down_url = get_setting('TASK_URL')['DOWNLOAD']
                            page.goto(down_url)
                            page.wait_for_load_state()
                        # =========================      营销场景数据写入数据库      =========================#
                        result_yxcj[store] = False
                        self.status_signal.emit("写入营销场景数据...")
                        if not task_status[store]['YXCJ'] and yxcj_file_name_value != '':
                            try:
                                report_row = page.locator(f'tr:has-text("{yxcj_file_name_value}")')
                                # 等待该行的状态变为 "生成成功"
                                success_status = report_row.locator('span.color-green:has-text("生成成功")')
                                success_status.wait_for(timeout=120000)  # 等待最多 120 秒
                                success_status.click(timeout=30000)

                                line_open_tr = page.wait_for_selector('tr[mx-stickytable-operation="line-open"]',
                                                                      timeout=5000)

                                # 找到这一行里的下载按钮并点击
                                download_button = line_open_tr.query_selector('button:has-text("下载")')

                                # 监听下载事件
                                with page.expect_download() as download_info:
                                    download_button.click()

                                # 获取下载的压缩文件对象
                                download = download_info.value

                                # 获取下载文件的建议文件名
                                suggested_filename = download.suggested_filename
                                # 保存下载的文件到指定路径
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_yxcj = self.get_file_to_pd(download_folder, suggested_filename)
                                columns_to_drop = ['自然流量转化金额', '自然流量曝光量']
                                df_yxcj = df_yxcj.drop(
                                    columns=[col for col in columns_to_drop if col in df_yxcj.columns])
                                df_yxcj['日期'] = to_datetime(df_yxcj['日期'])
                                if not df_yxcj.empty:
                                    threading.Thread(target=auto_data_to_mysql,
                                                     args=(df_yxcj, 'ecapp_yxcj', store, dburl, None, None, None, today,
                                                           task_status)).start()
                                    task_status[store]['YXCJ'] = True
                                else:
                                    task_status[store]['YXCJ'] = True
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--营销场景写入失败:' + str(e)
                                self.logger.error(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      计划数据写入数据库 不再下载计划表 2025-03-24     =========================#
                        # self.status_signal.emit("写入计划数据...")
                        if 1 == 2 and not task_status[store]['JH'] and jh_file_name_value != '':
                            try:
                                report_row_jh = page.locator(f'tr:has-text("{jh_file_name_value}")')
                                success_status_jh = report_row_jh.locator('span.color-green:has-text("生成成功")')
                                success_status_jh.wait_for(timeout=60000)  # 等待最多 60 秒
                                success_status_jh.click(timeout=30000)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                line_open_tr = page.wait_for_selector('tr[mx-stickytable-operation="line-open"]',
                                                                      timeout=5000)
                                download_button = line_open_tr.query_selector('button:has-text("下载")')
                                # 监听下载事件
                                with page.expect_download() as download_info:
                                    download_button.click()

                                download = download_info.value
                                suggested_filename = download.suggested_filename

                                # 保存下载的文件到指定路径
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_jh = self.get_file_to_pd(download_folder, suggested_filename)
                                task_status[store]['JH'] = True
                                if not df_jh.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_jh, 'ecapp_jh', store, dburl, None, None, None, today, task_status)).start()
                                else:
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--计划数据写入失败:' + str(e)
                                self.logger.error(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      3.数据总览-日      =========================#
                        self.status_signal.emit("处理数据总览...")
                        if not task_status[store]['SJZLR']:
                            try:
                                url = get_setting('TASK_URL')['SJZLR']
                                page.goto(url)
                                # 等待页面加载
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                checkV = page.locator('span:has-text("体验新版")')
                                if checkV.count() > 0:
                                    checkV.click()
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 侦测弹框并取消或删除
                                page.on("dialog", lambda dialog: dialog.dismiss())
                                page.keyboard.press('Escape')
                                watermark_element = page.query_selector("#watermark-container")
                                if watermark_element:
                                    # 如果元素存在，使用 JavaScript 移除它
                                    page.evaluate("""
                                                () => {
                                                    const element = document.getElementById('watermark-container');
                                                    if (element) {
                                                        element.remove();
                                                    }
                                                }
                                            """)

                                # button = page.locator('button.ant-btn:has-text("查看表格")')
                                button = page.locator('button.ant-btn:has-text("查看所有指标")')
                                if button.is_visible():
                                    button.click(timeout=5000)
                                    page.wait_for_timeout(3000)
                                else:
                                    try:
                                        # 如果按钮不可见，先点击小屏幕按钮
                                        small_button = page.locator('a:has(span.small-than-1400)').first
                                        small_button.click()
                                    except:
                                        page.keyboard.press('Escape')
                                        small_button = page.locator('a:has(span.small-than-1400)').first
                                        small_button.click()

                                btn = page.locator("span", has_text="表格").last
                                btn.click(timeout=5000)

                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # df_sjzlr = self.get_sjzlr(page)

                                table = page.locator('div.ant-table-body table')
                                # 确保表格加载完成
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 只获取前7天的表头日期
                                date_headers = table.locator('thead th:not(:first-child) .dateLabel')
                                dates = [date_headers.nth(i).inner_text().strip() + " 00:00:00" for i in
                                         range(min(7, date_headers.count()))]

                                data = []
                                rows = table.locator('tbody tr')
                                for row_idx in range(rows.count()):
                                    row = rows.nth(row_idx)
                                    # 获取指标名称
                                    metric_name = row.locator('td:first-child .leading.columnTd').inner_text().strip()

                                    # 获取前7个数值
                                    cells = row.locator('td:not(:first-child) .value')
                                    values = [self.parse_value(cells.nth(i).inner_text()) for i in
                                              range(min(7, cells.count()))]

                                    if len(values) == len(dates):
                                        data.append({'指标名称': metric_name, **dict(zip(dates, values))})

                                df = DataFrame(data).set_index('指标名称').T
                                df.index.name = '日期'
                                df.insert(0, '类别', '本店')
                                # **新增列**
                                new_columns1 = ["支付老买家数", "回访成交", "咨询人数", "内容查看人数",
                                                "有过已购状态的客户数", "旺旺满意度"]
                                new_columns2 = ["48小时揽收及时率", "关联商品数", "总内容供给规模量", "纠纷投诉商责率",
                                                "商品覆盖率",
                                                "开播时长(小时)", "询单转化率", "老买家支付金额", "进入公域视频数",
                                                "总支付子订单数", "停留时长"]
                                df[new_columns1] = None
                                if "支付金额" in df.columns:
                                    df["总支付金额"] = df["支付金额"]
                                df[new_columns2] = None
                                res_sjzlr = df.reset_index()  # 去掉索引列
                                verbose_name_list = [
                                    '日期', '类别', '支付金额', '店铺客户数', '平均停留时长', '支付买家数',
                                    '支付转化率', '客单价',
                                    '老客复购率', '老客复购人数', '老客复购金额', '支付子订单数', '支付件数', '访客数',
                                    '浏览量',
                                    '退款金额（完结时间）', '净支付金额', '加购件数', '加购人数', '商品收藏人数', '咨询率',
                                    '旺旺人工响应时长(秒)',
                                    '24小时揽收及时率', '物流到货时长(小时)', '退款处理时长(天)', '成功退款率',
                                    '平台判责率', '全站推广花费',
                                    '关键词推广花费', '精准人群推广花费', '智能场景花费', '淘宝客佣金', '支付老买家数',
                                    '回访成交', '咨询人数',
                                    '内容查看人数', '有过已购状态的客户数', '旺旺满意度', '总支付金额',
                                    '48小时揽收及时率', '关联商品数',
                                    '总内容供给规模量', '纠纷投诉商责率', '商品覆盖率', '开播时长(小时)', '询单转化率',
                                    '老买家支付金额',
                                    '进入公域视频数', '总支付子订单数', '停留时长'
                                ]

                                # 创建空的 DataFrame
                                df_sjzlr = DataFrame(columns=verbose_name_list)
                                common_columns = df_sjzlr.columns.intersection(res_sjzlr.columns)
                                df_sjzlr[common_columns] = res_sjzlr[common_columns]

                                # # 增加处理下一步弹窗 2025-01-22=======>不再使用店透视插件  2025-03-05
                                # try:
                                #     modal = page.locator("div.guide-modal")
                                #     if modal.is_visible():
                                #         close_button = modal.locator("div.guide-modal-close-icon")
                                #         close_button.click()
                                # except:
                                #     pass
                                #
                                # try:
                                #     if page.locator("#ActiveEggModal").count() > 0:
                                #         # 如果存在活动突破数据弹窗，点击关闭按钮
                                #         close_button = page.locator(".sycm-active-egg-modal-content-close")
                                #         if close_button.count() > 0:
                                #             close_button.click()
                                # except:
                                #     page.goto(url)
                                #     page.wait_for_load_state()
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                # while True:
                                #     try:
                                #         page.wait_for_selector("#Dts_PortalHomeOverviewNew", timeout=5000)
                                #         break
                                #     except:
                                #         try:
                                #             element = page.wait_for_selector('div.ebase-frame-header-switch.hasActivity',
                                #                                              timeout=10000)
                                #             text = element.query_selector('span.ebase-frame-header-text').inner_text()
                                #             # 检查文本是否为 "体验新版"
                                #             if text == "体验新版":
                                #                 element.click()  # 点击元素
                                #                 page.wait_for_load_state()
                                #         except:
                                #             pass
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                # # 'button:has-text("自定义")'找到自定义按钮，'..'代表找到这个按钮的父级元素(div)
                                # custom_button_div = page.locator("button:has-text('自定义')").locator('..')
                                # # 在该 div 中找到包含 "日" 字样的按钮并点击
                                # custom_button_div.locator("button:has-text('日')").click()
                                # time.sleep(random.randint(3, 5))
                                # page.locator("#Dts_PortalHomeOverviewNew").click()
                                # while True:
                                #     try:
                                #         page.wait_for_selector('.el-dialog__wrapper', timeout=5000)
                                #         title_locator = page.locator(".el-dialog__title").first
                                #         if title_locator.count() > 0:
                                #             title_text = title_locator.inner_text()
                                #             if title_text == "升级提示 - 店透视":
                                #                 page.keyboard.press('Escape')
                                #                 page.locator("#Dts_PortalHomeOverviewNew").click()
                                #             else:
                                #                 break
                                #     except BaseException as e:
                                #         page.goto(url)
                                #         # 等待页面加载
                                #         page.wait_for_load_state()
                                #         page.locator("#Dts_PortalHomeOverviewNew").click()
                                #         pass
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                # # 找到“导出表格”按钮
                                # export_button = page.get_by_role("button", name="导出表格")
                                # # 找到该按钮所在的父元素
                                # export_button_div = export_button.locator('..')
                                # # 在该父元素中找到下拉箭头按钮
                                # dropdown_button = export_button_div.locator('i.el-icon-arrow-down')
                                # dropdown_button.scroll_into_view_if_needed()
                                # # 点击下拉按钮
                                # dropdown_button.click()
                                # # 点击导出表格 csv(所有列) 选项
                                # export_option = page.locator('li:has-text("导出表格 csv(所有列)")')
                                #
                                # # 监听下载事件
                                # with page.expect_download() as download_info:
                                #     export_option.click()
                                #
                                # download = download_info.value
                                # suggested_filename = download.suggested_filename
                                #
                                # # 保存下载的文件到指定路径
                                # download.save_as(os.path.join(download_folder, suggested_filename))
                                # df_sjzlr = self.get_file_to_pd(download_folder, suggested_filename)

                                # page.keyboard.press('Escape')
                                if not df_sjzlr.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_sjzlr, 'ecapp_sjzlr', store, dburl, None, None, None, today,
                                        task_status)).start()
                                    task_status[store]['SJZLR'] = True
                                else:
                                    task_status[store]['SJZLR'] = True
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                                    self.logger.warning('--' + store + '--数据总览-日数据为空')
                                store_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--数据总览-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      4.店铺流量来源-日      =========================#
                        self.status_signal.emit("处理店铺流量来源...")
                        if not task_status[store]['LLLYDPR']:
                            try:
                                url = get_setting('TASK_URL')[
                                          'LLLYDPR'] + yesterday + '%7C' + yesterday + '&dateType=day&device=2'
                                page.goto(url)
                                # 等待页面加载
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                page.on("dialog", lambda dialog: dialog.dismiss())
                                # 增加去掉遮罩新手引导
                                try:
                                    mask = page.locator('div.flow-guide-mask')
                                    # 持续点击直到该元素消失
                                    while mask.count() > 0:
                                        mask.click(force=True)
                                        # 等待一段时间以确保页面反应
                                        time.sleep(random.randint(self.min_wait, self.max_wait))
                                except:
                                    pass
                                try:
                                    if page.locator("#ActiveEggModal").count() > 0:
                                        # 如果存在活动突破数据弹窗，点击关闭按钮
                                        close_button = page.locator(".sycm-active-egg-modal-content-close")
                                        if close_button.count() > 0:
                                            close_button.click()
                                except:
                                    page.goto(url)
                                    page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))

                                try:
                                    # 查找“返回旧版”按钮
                                    return_old_version_button = page.query_selector('div.flow2024-switch.to-old')
                                    text_content = return_old_version_button.text_content()
                                    if '返回旧版' in text_content:
                                        return_old_version_button.click()
                                        time.sleep(random.randint(self.min_wait, self.max_wait))
                                    else:
                                        print('没有“返回旧版”按钮直接下载数据')
                                except:
                                    pass
                                page.keyboard.press('Escape')
                                export_option = page.locator('span:has-text("下载")')
                                # while True:
                                #     try:
                                #         page.wait_for_selector("#Dts_FlowMonitorShopsourceSelfRouter", timeout=5000)
                                #         break
                                #     except:
                                #         try:
                                #             element = page.wait_for_selector('div.ebase-frame-header-switch.hasActivity',
                                #                                              timeout=10000)
                                #         except:
                                #             pass
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                #
                                # page.locator("#Dts_FlowMonitorShopsourceSelfRouter").first.click()
                                # while True:
                                #     try:
                                #         page.wait_for_selector('.el-dialog__wrapper', timeout=5000)
                                #         title_locator = page.locator(".el-dialog__title").first
                                #         if title_locator.count() > 0:
                                #             title_text = title_locator.inner_text()
                                #             if title_text == "升级提示 - 店透视":
                                #                 page.keyboard.press('Escape')
                                #                 page.locator("#Dts_FlowMonitorShopsourceSelfRouter").first.click()
                                #             else:
                                #                 break
                                #     except BaseException as e:
                                #         page.goto(url)
                                #         # 等待页面加载
                                #         page.wait_for_load_state()
                                #         page.locator("#Dts_FlowMonitorShopsourceSelfRouter").click()
                                #         pass
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                # # 找到“导出表格”按钮
                                # export_button = page.get_by_role("button", name="导出表格")
                                # # 找到该按钮所在的父元素
                                # export_button_div = export_button.locator('..')
                                # # 在该父元素中找到下拉箭头按钮
                                # dropdown_button = export_button_div.locator('i.el-icon-arrow-down')
                                # dropdown_button.scroll_into_view_if_needed()
                                # # 点击下拉按钮
                                # dropdown_button.click()
                                # # 点击导出表格 csv(所有列) 选项
                                # export_option = page.locator('li:has-text("导出表格 csv(所有列)")')

                                # 监听下载事件
                                with page.expect_download() as download_info:
                                    export_option.click()

                                download = download_info.value
                                suggested_filename = download.suggested_filename

                                # 保存下载的文件到指定路径
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_lllydpr = self.get_file_to_pd(download_folder, suggested_filename)
                                df_lllydpr = df_lllydpr.drop(index=[0, 1, 2, 3])

                                # 将第四行的值作为列名
                                new_columns = df_lllydpr.iloc[0]
                                df_lllydpr = df_lllydpr[1:]  # 删除第四行
                                df_lllydpr.columns = new_columns
                                df_lllydpr = df_lllydpr[(df_lllydpr['二级来源'].isin(['手淘推荐', '手淘搜索'])) & (
                                        df_lllydpr['三级来源'] == '汇总')]

                                # 在最前面增加一列 '流量来源'
                                df_lllydpr.insert(0, '流量来源', df_lllydpr['一级来源'] + '-' + df_lllydpr['二级来源'])
                                df_lllydpr['三级来源'] = '-'
                                df_lllydpr.insert(df_lllydpr.columns.get_loc('三级来源') + 1, '日期', yesterday)
                                columns_to_keep = [
                                    '流量来源', '一级来源', '二级来源', '三级来源', '日期', '访客数',
                                    '引导短视频访客数',
                                    '引导商品访客数', '新访客', '引导店铺页访客数', '加购人数', '收藏商品买家数',
                                    '关注店铺买家数', '下单买家数', '下单转化率', '下单金额', '支付金额', '支付买家数',
                                    '支付转化率', '客单价', 'UV价值', '直接支付买家数', '收藏商品-支付买家数',
                                    '粉丝支付买家数', '加购商品-支付买家数', '种草成交人数', '种草成交金额'
                                ]

                                df_lllydpr = df_lllydpr[columns_to_keep]
                                df_lllydpr.rename(
                                    columns={'收藏商品买家数': '收藏人数', '关注店铺买家数': '关注店铺人数',
                                             '收藏商品-支付买家数': '收藏支付买家数', '新访客': '新访客数',
                                             '加购商品-支付买家数': '加购支付买家数'}, inplace=True)
                                df_lllydpr.insert(df_lllydpr.columns.get_loc('收藏人数') + 1, '总收藏加购人数', None)
                                df_lllydpr.insert(df_lllydpr.columns.get_loc('总收藏加购人数') + 1, '收藏率', None)
                                df_lllydpr.insert(df_lllydpr.columns.get_loc('收藏率') + 1, '加购率', None)
                                df_lllydpr.insert(df_lllydpr.columns.get_loc('加购率') + 1, '收藏加购率', None)

                                # page.keyboard.press('Escape')
                                if not df_lllydpr.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_lllydpr, 'ecapp_lllydpr', store, dburl, None, None, None, today,
                                        task_status)).start()
                                    task_status[store]['LLLYDPR'] = True
                                else:
                                    task_status[store]['LLLYDPR'] = True
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                                store_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--流量来源店铺-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      5.全商品核心概况-日      =========================#
                        tmhxgkr_flag = True  # 追踪的天猫商品的HXGKR表是否全是True,全是True返回True,否则重新下载一次全商品核心概况
                        for proId in product_list:
                            if not task_status[store][proId]['HXGKR']:
                                tmhxgkr_flag = False
                                break
                        if not tmhxgkr_flag:
                            try:
                                hxgkr_url = get_setting('TASK_URL')[
                                                'HXGKR'] + yesterday + '%7C' + yesterday + '&dateType=day'
                                page.goto(hxgkr_url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                try:
                                    splitBtn = page.locator("span:has-text('拆分视角')").first
                                    splitBtn.click(timeout=3000, force=True)
                                except BaseException as e:
                                    pass

                                # 找到下载按钮
                                export_option = page.locator("a.sycm-traced-download-text")
                                export_option.scroll_into_view_if_needed()
                                # 处理可能的遮挡元素
                                page.evaluate("document.querySelector('#DTS_ENTRY__sycm')?.remove()")  # 移除拦截的元素
                                # 先 hover 再点击
                                export_option.hover()
                                # 监听下载事件
                                with page.expect_download() as download_info:
                                    export_option.click()

                                download = download_info.value
                                suggested_filename = download.suggested_filename

                                # 保存下载的文件到指定路径
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_hxgkr_file = self.get_file_to_pd(download_folder, suggested_filename)
                                df_hxgkr_file = df_hxgkr_file.drop(index=[0, 1, 2])
                                new_columns = df_hxgkr_file.iloc[0]
                                df_hxgkr_file = df_hxgkr_file[1:]  # 删除
                                df_hxgkr_file.columns = new_columns

                                hxgkr_columns = ['商品名称', '商品ID', '商品图片', '商品类型', '统计日期', '商品访客数',
                                                 '商品微详情访客数', '商品浏览量', '平均停留时长', '商品详情页跳出率',
                                                 '商品加购件数',
                                                 '商品加购人数', '商品收藏人数', '访问加购转化率', '访问收藏转化率',
                                                 '下单买家数', '下单件数', '下单金额', '下单转化率', '支付买家数',
                                                 '支付件数',
                                                 '支付金额', '商品支付转化率', '聚划算支付金额', '支付新买家数',
                                                 '支付老买家数', '老买家支付金额', 'UV价值', '人均浏览量',
                                                 '详情页跳出人数', '聚划算支付金额占比', '件单价', '客单价',
                                                 '成功退款金额']
                                df_hxgkr = DataFrame(columns=hxgkr_columns)
                                df_hxgkr = df_hxgkr_file.reindex(columns=hxgkr_columns, fill_value=None)
                                df_hxgkr = df_hxgkr[df_hxgkr['商品ID'].isin(product_list)]  # 仅保留需要的商品的数据
                                df_hxgkr['商品名称'] = ''
                                for proId in product_list:
                                    task_status[store][proId]['HXGKR'] = True
                                if not df_hxgkr.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_hxgkr, 'ecapp_hxgkr', store, dburl, None, store_type[store], product_list, today,
                                        task_status)).start()
                                else:
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                                pro_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--全商品核心概况-日获取失败:' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      6.主体报表-关键词推广,人群推广,货品运营,全站推广-日   =========================#
                        self.status_signal.emit("处理主体表数据...")
                        result_ZTR = []
                        ztr_dict = {'关键词推广': 'subway', '人群推广': 'yinli', '货品运营': 'hpyy', '全站推广': 'qztg'}
                        file_name_dict = {'关键词推广': '', '人群推广': '', '货品运营': '', '全站推广': ''}
                        if task_status[store]['ZTR_GJ']:
                            result_ZTR.append('关键词推广')
                        if task_status[store]['ZTR_RQ']:
                            result_ZTR.append('人群推广')
                        if task_status[store]['ZTR_HP']:
                            result_ZTR.append('货品运营')
                        if task_status[store]['ZTR_QZ']:
                            result_ZTR.append('全站推广')
                        # 过滤已下载过的数据
                        ztr_dict_new = {k: v for k, v in ztr_dict.items() if k not in result_ZTR}
                        ztr_main_url = get_setting('TASK_URL')['ZTR']
                        for okey, curTemplateId in ztr_dict_new.items():
                            if okey == '关键词推广':
                                tableName = 'ZTR_GJ'
                            elif okey == '人群推广':
                                tableName = 'ZTR_RQ'
                            elif okey == '货品运营':
                                tableName = 'ZTR_HP'
                            else:
                                tableName = 'ZTR_QZ'
                            if not task_status[store][tableName]:
                                try:
                                    ztr_url = ztr_main_url + curTemplateId + '&startTime=' + yesterday15 + '&endTime=' + yesterday + '&splitType=day'
                                    page.goto(ztr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    while True:
                                        try:
                                            button_locator = page.locator('button:has-text("下载报表")')
                                            button_locator.scroll_into_view_if_needed()
                                            button_locator.click()
                                            break
                                        except:
                                            time.sleep(random.randint(self.min_wait, self.max_wait))
                                            page.goto(ztr_url)
                                            page.wait_for_load_state()
                                            button_locator = page.locator('button:has-text("下载报表")')
                                            button_locator.scroll_into_view_if_needed()
                                            button_locator.click()

                                    div_dialog_body = page.locator('div.dialog-body').last
                                    # 获取该元素的父级 div
                                    div_dialog = div_dialog_body.locator('..')
                                    # file_name_input = div_dialog.locator('input[value^="主体报表_"]')颐真,POMAC店主体报表名称变更问题20250623
                                    file_name_input = div_dialog.locator('input[value^="主体报表_"], input[value^="商品报表_"]')
                                    file_name = file_name_input.input_value()
                                    file_name_dict[okey] = file_name
                                    # ztr_gjc_file_name_value = file_name_input.input_value()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 下载按钮
                                    div_dialog.locator('button:has-text("确定")').click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                except BaseException as e:
                                    proMsg = '--' + store + f'--主体报表-{okey}数据获取失败:' + str(e)
                                    self.logger.error(proMsg)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                        if any(value != '' for value in file_name_dict.values()):
                            # 点完下载后转到下载页面
                            down_url = get_setting('TASK_URL')['DOWNLOAD']
                            page.goto(down_url)
                            page.wait_for_load_state()

                            for key, fileName in file_name_dict.items():
                                if fileName != '':
                                    if key == '关键词推广':
                                        tableName = 'ZTR_GJ'
                                    elif key == '人群推广':
                                        tableName = 'ZTR_RQ'
                                    elif key == '货品运营':
                                        tableName = 'ZTR_HP'
                                    else:
                                        tableName = 'ZTR_QZ'
                                    try:
                                        report_row = page.locator(f'tr:has-text("{fileName}")')
                                        # 等待该行的状态变为 "生成成功"
                                        success_status = report_row.locator('span.color-green:has-text("生成成功")')
                                        success_status.wait_for(timeout=180000)  # 等待最多 180 秒
                                        success_status.click(timeout=30000)

                                        line_open_tr = page.wait_for_selector(
                                            'tr[mx-stickytable-operation="line-open"]',
                                            timeout=5000)

                                        # 找到这一行里的下载按钮并点击
                                        download_button = line_open_tr.query_selector('button:has-text("下载")')

                                        # 监听下载事件
                                        with page.expect_download() as download_info:
                                            download_button.click()

                                        # 获取下载的压缩文件对象
                                        download = download_info.value

                                        # 获取下载文件的建议文件名
                                        suggested_filename = download.suggested_filename
                                        # 保存下载的文件到指定路径
                                        download.save_as(os.path.join(download_folder, suggested_filename))
                                        df_ztr = self.get_file_to_pd(download_folder, suggested_filename)
                                        verbose_name_list = [
                                            '日期', '主体ID', '主体类型', '主体名称', '展现量', '点击量', '花费',
                                            '点击率', '平均点击花费', '千次展现花费',
                                            '总预售成交金额', '总预售成交笔数', '直接预售成交金额', '直接预售成交笔数',
                                            '间接预售成交金额', '间接预售成交笔数', '直接成交金额', '间接成交金额',
                                            '总成交金额', '总成交笔数',
                                            '直接成交笔数', '间接成交笔数', '点击转化率', '投入产出比', '总成交成本',
                                            '总购物车数', '直接购物车数', '间接购物车数', '加购率', '收藏宝贝数',
                                            '收藏店铺数', '店铺收藏成本', '总收藏加购数', '总收藏加购成本',
                                            '宝贝收藏加购数', '宝贝收藏加购成本', '总收藏数', '宝贝收藏成本',
                                            '宝贝收藏率', '加购成本',
                                            '拍下订单笔数', '拍下订单金额', '直接收藏宝贝数', '间接收藏宝贝数',
                                            '优惠券领取量', '购物金充值笔数', '购物金充值金额', '旺旺咨询量',
                                            '引导访问量', '引导访问人数',
                                            '引导访问潜客数', '引导访问潜客占比', '入会率', '入会量', '引导访问率',
                                            '深度访问量', '平均访问页面数', '成交新客数', '成交新客占比',
                                            '会员首购人数',
                                            '会员成交金额', '会员成交笔数', '成交人数', '人均成交笔数', '人均成交金额',
                                            '自然流量转化金额', '自然流量曝光量'
                                        ]
                                        # 保留 verbose_name_list 中的列
                                        df_ztr_filtered = df_ztr[verbose_name_list]
                                        df_ztr_filtered['subject_name'] = ''
                                        task_status[store][tableName] = True
                                        if not df_ztr_filtered.empty:
                                            threading.Thread(target=auto_data_to_mysql,
                                                             args=(
                                                                 df_ztr_filtered, 'ecapp_ztr', store, dburl, key, None,
                                                                 None, today, task_status)).start()
                                        else:
                                            threading.Thread(target=only_up_task_status,
                                                             args=(dburl, today, task_status)).start()

                                        store_flag = True
                                    except BaseException as e:
                                        proMsg = '--' + store + f'--主体报表-{key}写入失败:' + str(e)
                                        self.logger.error(proMsg)
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                        # 把商品数据的获取移到if外面去
                    elif store_type[store] == '京东':
                        proMsg = '--' + store + '开始检查/获取店铺级数据'
                        self.logger.info(proMsg)
                        # =========================      1. 京东-数据总览-日      =========================#
                        self.status_signal.emit("处理数据总览...")
                        if not task_status[store]['SJZLR'] and store == '青岛昌隆文具有限公司':
                            try:
                                url = get_setting('TASK_URL')['JDSJZLR']
                                page.goto(url)
                                # 等待页面加载
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 检查登录数据是否过期
                                login_btn = page.locator("div.btn-wrapper.headv a.login-btn")

                                if login_btn.count() > 0:
                                    btn_text = login_btn.text_content().strip()
                                    if "登录" in btn_text:
                                        proMsg = '--' + store + '登录数据已过期,请重新保存登录数据后再重新下载本店铺数据!'
                                        self.logger.error(proMsg)
                                        try:
                                            context.close()
                                        except:
                                            pass
                                        try:
                                            browser.close()
                                        except:
                                            pass
                                        continue
                                self.remove_jd_ad(page)
                                # 京东商智订单到期提醒弹窗
                                close_btn = page.locator("span.r-modal-close")
                                if close_btn.count() > 0:
                                    close_btn.click()
                                #   京东用自营店下载自营和POP的数据,POP的数据计入白雪文具官方旗舰店==>POP的 单独下载了
                                # store_category_list = ['自营', 'POP']
                                store_category_list = ['自营']
                                jd_store = store
                                for store_cate in store_category_list:
                                    if store_cate == 'POP':
                                        jd_store = '白雪文具官方旗舰店'
                                    # 1. 根据 ID 定位元素
                                    margin_div = page.locator("#Margin1")

                                    # 2. 获取父级 div
                                    parent_div = margin_div.locator("xpath=..")

                                    # 3. 点经营模式,选自营
                                    target_div = parent_div.locator(
                                        "div:nth-child(3) > div.jmtd-form-item-content")  # 精确选择第三个 div
                                    target_div.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    target_element = page.locator(
                                        f"//div[@class='jmtd-dropdown-content']//div[@class='jmtd-label' and text()='{store_cate}']")
                                    target_element.first.click()  # 经营模式选自营
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 日期默认就是昨天,所以不需要设置
                                    # 4. 点 "确定" 文本的按钮
                                    target_button = parent_div.locator(
                                        "div.jmtd-form-item-content >> button:has-text('确定')")
                                    target_button.click()  # 点击包含 "确定" 文本的按钮
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 找到 id 为 ContentCard 的 div
                                    content_card_div = page.locator("#ContentCard")
                                    # 5. 点 "下载数据" 按钮
                                    download_button = content_card_div.locator("button:has-text('下载数据')")
                                    download_button.click()  # 点击下载数据按钮
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 6. 点 "汇总下载"
                                    sum_download_button = page.locator(
                                        "div.jmtd-dropdown-content >> div.jmtd-label:has-text('汇总下载')")
                                    # 监听下载事件
                                    with page.expect_download() as download_info:
                                        sum_download_button.click()

                                    download = download_info.value
                                    suggested_filename = download.suggested_filename

                                    # 7.保存下载的文件到指定路径并读取
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_jdsjzlr = self.get_file_to_pd(download_folder, suggested_filename)
                                    df_jdsjzlr.rename(columns={
                                        '成交人数': '支付买家数', '成交转化率': '支付转化率',
                                        '成交单量': '支付子订单数', '成交商品件数': '支付件数',
                                        '成交金额': '支付金额', '成交客单价': '客单价', }, inplace=True)
                                    df_jdsjzlr['日期'] = yesterday0
                                    # 日期后增加一列 '类别'，值为 '本店'
                                    df_jdsjzlr.insert(df_jdsjzlr.columns.get_loc('日期') + 1, '类别', '本店')
                                    # task_status[jd_store]['JH'] = True  # 京东不需要下载JH表
                                    if not df_jdsjzlr.empty:
                                        threading.Thread(target=auto_data_to_mysql, args=(df_jdsjzlr, 'ecapp_sjzlr', jd_store, dburl, None, '京东', None, today,task_status)).start()
                                        task_status[jd_store]['SJZLR'] = True
                                    else:
                                        task_status[jd_store]['SJZLR'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                                    task_status[jd_store]['SJZLR'] = True
                            except BaseException as e:
                                proMsg = '--' + store + '--数据总览-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        elif not task_status[store]['SJZLR'] and store == '白雪文具官方旗舰店':
                            try:
                                url = get_setting('TASK_URL')['JDSJZLRPOP']
                                page.goto(url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                down_btn = page.locator('div.grace-download-btn').first
                                with page.expect_download() as download_info:
                                    down_btn.click()

                                download = download_info.value
                                suggested_filename = download.suggested_filename
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_read = self.get_file_to_pd(download_folder, suggested_filename)
                                columns_to_keep = ['日期', '浏览量', '访客数', '成交单量','成交转化率', '成交金额','客单价', '成交商品件数', '成交客户数']

                                verbose_name_list = ['日期', '类别','浏览量','访客数', '成交客户数','成交转化率', '成交单量', '成交商品件数','成交金额', '客单价']
                                # 创建空的 DataFrame
                                df_sjzlr_pop = DataFrame(columns=verbose_name_list)
                                df_sjzlr_pop[columns_to_keep] = df_read[columns_to_keep]
                                df_sjzlr_pop['日期'] = yesterday0
                                df_sjzlr_pop['类别'] = '本店'
                                if not df_sjzlr_pop.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(df_sjzlr_pop, 'ecapp_sjzlr', store, dburl, None, '京东', None, today,task_status)).start()
                                    task_status[store]['SJZLR'] = True
                                else:
                                    task_status[store]['SJZLR'] = True
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                                task_status[store]['SJZLR'] = True
                            except BaseException as e:
                                proMsg = '--' + store + '--数据总览-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                        # =========================      2. 京东-店铺流量-日      =========================#
                        self.status_signal.emit("处理店铺流量数据...")
                        if not task_status[store]['LLLYDPR'] and store == '青岛昌隆文具有限公司':
                            try:
                                url = get_setting('TASK_URL')['JDLLLYDPR']
                                page.goto(url)
                                # 等待页面加载
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                self.remove_jd_ad(page)
                                #   京东用自营店下载自营和POP的数据,POP的数据计入白雪文具官方旗舰店==>POP改为单独下载
                                # store_category_list = ['自营', 'POP']
                                store_category_list = ['自营']
                                jd_store = store
                                for store_cate in store_category_list:
                                    if store_cate == 'POP':
                                        jd_store = '白雪文具官方旗舰店'
                                    # 1. 根据 ID 定位元素
                                    margin_div = page.locator("#Card1")
                                    # 3. 点经营模式,选自营
                                    target_div = margin_div.locator(
                                        "div:nth-child(3) > div.jmtd-form-item-content")  # 精确选择第三个 div
                                    target_div.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    target_element = page.locator(
                                        f"//div[@class='jmtd-dropdown-content']//div[@class='jmtd-label' and text()='{store_cate}']")
                                    target_element.first.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 日期默认就是昨天,所以不需要设置
                                    # 4. 点 "确定" 文本的按钮
                                    target_button = margin_div.locator(
                                        "div.jmtd-form-item-content >> button:has-text('确定')")
                                    target_button.click()  # 点击包含 "确定" 文本的按钮
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 找到 id 为 Row1 的 div
                                    content_card_div = page.locator("#Row1")
                                    # 5. 点 "下载数据" 按钮
                                    download_button = content_card_div.locator("button:has-text('下载数据')")
                                    download_button.click()  # 点击下载数据按钮
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 6. 点 "汇总下载"
                                    sum_download_button = page.locator("div.jmtd-dropdown-content:has-text('汇总下载')")
                                    sum_download_button.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 获取所有的jmtd-modal元素
                                    # 获取页面上的所有class为'jmtd-label'且文本为'一级渠道'的div元素
                                    channels = page.locator('div.jmtd-label:has-text("一级渠道")')

                                    # 获取第三个'一级渠道' div
                                    third_channel = channels.nth(2)
                                    third_channel.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    page.locator("div.jmtd-label:has-text('二级渠道')").click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 获取页面上所有 "确定" 按钮
                                    confirm_buttons = page.locator('button:has-text("确定")')
                                    # 获取最后一个 "确定" 按钮
                                    last_confirm_button = confirm_buttons.nth(confirm_buttons.count() - 1)

                                    # 监听下载事件
                                    with page.expect_download() as download_info:
                                        last_confirm_button.click()

                                    download = download_info.value
                                    suggested_filename = download.suggested_filename

                                    # 7.保存下载的文件到指定路径并读取
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_jdlllydpr = self.get_file_to_pd(download_folder, suggested_filename)
                                    df_jdlllydpr = df_jdlllydpr.copy()
                                    df_jdlllydpr = df_jdlllydpr[df_jdlllydpr['二级渠道'].isin(['搜索', '推荐'])]

                                    df_jdlllydpr['日期'] = yesterday0
                                    # 日期后增加列
                                    df_jdlllydpr.insert(df_jdlllydpr.columns.get_loc('日期') + 1, '流量来源',
                                                        '京东平台')
                                    df_jdlllydpr.insert(df_jdlllydpr.columns.get_loc('二级渠道') + 1, '三级来源', '-')
                                    # df_jdlllydpr = df_jdlllydpr.drop(['浏览量(PV)', '人均浏览量(人均PV)','平均停留时长(秒)','成交子单数'], axis=1)
                                    columns_to_drop = ['浏览量(PV)', '人均浏览量(人均PV)', '平均停留时长(秒)',
                                                       '成交子单数']
                                    existing_columns_to_drop = list(set(columns_to_drop) & set(df_jdlllydpr.columns))  # 获取存在的列
                                    df_jdlllydpr = df_jdlllydpr.drop(existing_columns_to_drop, axis=1)

                                    if not df_jdlllydpr.empty:
                                        threading.Thread(target=auto_data_to_mysql, args=(df_jdlllydpr, 'ecapp_lllydpr', jd_store, dburl, None, '京东', None, today,task_status)).start()
                                        task_status[jd_store]['LLLYDPR'] = True
                                    else:
                                        task_status[jd_store]['LLLYDPR'] = True
                                        threading.Thread(target=only_up_task_status, args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--店铺流量来源-日获取失败:' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        if not task_status[store]['LLLYDPR'] and store == '白雪文具官方旗舰店':
                            try:
                                url = get_setting('TASK_URL')['JDLLLYDPRPOP']
                                page.goto(url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                self.remove_jd_ad(page)
                                page.locator("div.jmtd-tag >> text=天").first.click(timeout=3000)
                                card = page.locator('div.flow-source-tab')  # 先定位整个来源渠道模块
                                card.locator('button:has-text("下载数据")').click(timeout=3000)
                                # 6. 点 "汇总下载"
                                sum_download_button = page.locator("div.jmtd-dropdown-content:has-text('汇总下载')")
                                sum_download_button.click()

                                channels = page.locator('//div[@class="jmtd-base-input-content"][.//div[text()="一级来源"]]')
                                channels.click(timeout=3000)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                page.locator("div.jmtd-label:has-text('二级来源')").click()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 获取页面上所有 "确定" 按钮
                                confirm_buttons = page.locator('button:has-text("确定")')
                                # 获取最后一个 "确定" 按钮
                                last_confirm_button = confirm_buttons.nth(confirm_buttons.count() - 1)

                                with page.expect_download() as download_info:
                                    last_confirm_button.click()

                                download = download_info.value
                                suggested_filename = download.suggested_filename

                                # 7.保存下载的文件到指定路径并读取
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_read_pop = self.get_file_to_pd(download_folder, suggested_filename)
                                df_read_pop = df_read_pop.copy()
                                df_read_pop = df_read_pop[df_read_pop['二级来源'].isin(['搜索', '推荐'])]
                                columns_to_keep = ['时间', '一级来源', '二级来源', '访客数', '加购客户数', '加购转化率','成交客户数',
                                                   '成交转化率', '成交金额', '客单价', 'uv价值','店铺关注人数','下单客户数','下单转化率','下单金额']

                                verbose_name_list = ['时间','流量来源', '一级来源', '二级来源', '三级来源',  '访客数', 'uv价值','客单价','加购客户数','店铺关注人数', '加购转化率', '下单客户数','下单转化率','下单金额','成交客户数', '成交转化率','成交金额']

                                # 创建空的 DataFrame
                                df_lllydpr_pop = DataFrame(columns=verbose_name_list)
                                df_lllydpr_pop[columns_to_keep] = df_read_pop[columns_to_keep]
                                df_lllydpr_pop['时间'] = yesterday0
                                df_lllydpr_pop['流量来源'] = '京东平台'
                                df_lllydpr_pop['三级来源'] = '-'
                                if not df_lllydpr_pop.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(df_lllydpr_pop, 'ecapp_lllydpr', store, dburl, None, '京东', None, today, task_status)).start()
                                    task_status[store]['LLLYDPR'] = True
                                else:
                                    task_status[store]['LLLYDPR'] = True
                                    threading.Thread(target=only_up_task_status, args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--店铺流量来源-日获取失败:' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      3. 京东-营销场景-日      =========================#
                        self.status_signal.emit("处理营销场景数据...")
                        if not task_status[store]['YXCJ']:
                            #=======================================  营销场景除全站营销外的数据取值位置给我京东商智-广告概况 20250904  =======================================#'JDYXCJRNEW'
                            if store == '白雪文具官方旗舰店':
                                jdyxcjrnew_url = get_setting('TASK_URL')['JDYXCJRNEWBX']
                            else:
                                jdyxcjrnew_url = get_setting('TASK_URL')['JDYXCJRNEW']
                            page.goto(jdyxcjrnew_url)
                            page.wait_for_load_state()
                            time.sleep(random.randint(self.min_wait, self.max_wait))

                            # # 1. 点击第6个日期输入框，弹出日历===平时下载时默认就是昨日数据，所以这里不需要点击
                            # date_picker = page.locator("div.jmtd-base-input-top").nth(5)
                            # date_picker.click()
                            #
                            # # 2. 确保弹出的日历面板出现
                            # panel = page.locator("div.jmtd-dropdown-panel")
                            # panel.wait_for()
                            #
                            # # 3.切换「按天」
                            # panel.locator(".jmtd-date-picker-switch-panel-item", has_text="按天").click(timeout=3000)
                            #
                            # # 5. 选择日期
                            # panel.locator(".jmtd-date-picker-cell-content-inner", has_text="2").first.click(timeout=3000)
                            # time.sleep(random.randint(self.min_wait, self.max_wait))
                            # page.locator("button[data-event-name='confirm']").click(timeout=3000)
                            # time.sleep(random.randint(self.min_wait, self.max_wait))
                            if store == '白雪文具官方旗舰店':
                                download_button  = page.locator("span.download-data:has-text('下载数据')").last
                            else:
                                download_button = page.query_selector("div.advert-grid button.bd-button-new.bd-button-v1.default.outline:has-text('下载数据')")
                            download_button.scroll_into_view_if_needed()

                            # 监听下载事件
                            with page.expect_download() as download_info:
                                download_button.click(timeout=3000)

                            # 获取下载的压缩文件对象
                            download = download_info.value

                            # 获取下载文件的建议文件名
                            suggested_filename = download.suggested_filename
                            # 保存下载的文件到指定路径
                            download.save_as(os.path.join(download_folder, suggested_filename))
                            df_jdyxcjrnew = self.get_file_to_pd(download_folder, suggested_filename)
                            df_jdyxcjrnew = df_jdyxcjrnew.copy()

                            if df_jdyxcjrnew.empty:
                                jdyxcjr_file_name_value = ''
                            else:
                                jdyxcjr_file_name_value = '京东营销数据已下载'

                            try:
                                jdyxcjr_url = get_setting('TASK_URL')['JDYXCJR']
                                page.goto(jdyxcjr_url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                try:
                                    self.remove_jd_ad(page)
                                    # 查找页面上是否有遮罩窗口 close-modal 元素
                                    close_modal = page.locator('span.close-modal')
                                    # 检查该元素是否存在并可见,点击关闭
                                    if close_modal.is_visible():
                                        close_modal.click()
                                except:
                                    pass
                                # if store == '白雪文具官方旗舰店':  # 选择广告PIN主体
                                # 选择广告PIN主体
                                try:
                                    authInput = page.locator('div:has(span.tip-label-name:has-text("账号"))').first
                                    if authInput.count() > 0:
                                        closeBtn = authInput.locator("i.jad-icon.jad-icon-close2")
                                        if closeBtn.count() > 0:
                                            closeBtn.click(timeout=3000)
                                except:
                                    pass

                                # # ===================================== 增加全站营销数据  2025-04-09 =====================================
                                # # 添加点击知道了按钮
                                # ok_btn = page.locator('button:has-text("知道了")')
                                # # ok_btn.wait_for(timeout=3000)
                                # # if ok_btn.is_visible():
                                # #     ok_btn.click()
                                # try:
                                #     ok_btn.click(timeout=5000)
                                # except:
                                #     pass
                                qzyxBtn = page.locator('span.jad-tabs-nav-tab-no-icon:has-text("全站营销")').last
                                qzyxBtn.click(timeout=3000)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))

                                # 1.点击日期,选择昨天
                                date_picker_div = page.locator('div.jad-input-wrapper.jad-date-picker-editor')
                                date_picker_div.click()
                                # 2. 等待日期选择弹窗出现
                                page.wait_for_selector(
                                    'div.jad-dropdown-popper.jad-click-popper.custom-date-range-popover')

                                # 3. 查找弹窗中的“昨天”选项并点击
                                yesterday_option = page.locator(
                                    'div.custom-date-range-popover div.jad-date-picker-shortcut')
                                yesterday_label = yesterday_option.locator('label:has-text("最近15天")')
                                yesterday_label.click()
                                # 点击 查询
                                target_button = page.locator("button:has-text('查询')").first
                                target_button.scroll_into_view_if_needed()
                                target_button.click()
                                page.wait_for_load_state()

                                downBtn = page.locator('button.download-report-icon').first
                                downBtn.click(timeout=3000)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                second_download_card = page.locator('div.download-card').last
                                jdyqzyx_file_name_value = second_download_card.locator('input.jad-input').input_value()
                                start_button = second_download_card.locator('button:has-text("开始生成")')
                                start_button.click()
                                time.sleep(random.randint(6, 8))
                                # ===================================== 增加全站营销数据结束  2025-04-09 =====================================
                                if jdyxcjr_file_name_value != '':

                                    columns = ['scene_id', 'scene_name', 'date', 'cost', 'impressions', 'clicks', 'avg_click_cost', 'cost_per_thousand_impressions',
                                               'click_through_rate', 'total_sales_orders', 'total_sales_amount', 'total_cart_count', 'add_to_cart_rate', 'click_conversion_rate', 'roi',
                                               'guided_potential_customer_rate']
                                    df_jdyxcjr = DataFrame(columns=columns)
                                    df_jdyxcjr['scene_id']=df_jdyxcjrnew['产品线']
                                    df_jdyxcjr['scene_name']=df_jdyxcjrnew['产品线']

                                    df_jdyxcjr['date'] = yesterday0
                                    # df_jdyxcjr['date'] = '2025-09-02 00:00:00'
                                    df_jdyxcjr['cost']=df_jdyxcjrnew['花费']
                                    df_jdyxcjr['impressions']=df_jdyxcjrnew['展现数']
                                    df_jdyxcjr['clicks']=df_jdyxcjrnew['点击数']
                                    df_jdyxcjr['avg_click_cost']=df_jdyxcjrnew['平均点击成本']
                                    df_jdyxcjr['cost_per_thousand_impressions']=df_jdyxcjrnew['平均千次展现成本']
                                    df_jdyxcjr['click_through_rate']=df_jdyxcjrnew['点击率']
                                    df_jdyxcjr['total_sales_amount']=df_jdyxcjrnew['总订单金额']
                                    df_jdyxcjr['total_cart_count']=df_jdyxcjrnew['总加购数']
                                    df_jdyxcjr['click_conversion_rate']=df_jdyxcjrnew['转化率']
                                    df_jdyxcjr['roi']=df_jdyxcjrnew['投产比']

                                    if jdyqzyx_file_name_value != '':  # 添加全站营销数据  2025-04-09
                                        report_row = page.locator(f'tr:has-text("{jdyqzyx_file_name_value}")').first
                                        success_status = report_row.locator(
                                            'span.download-report-font-green:has-text("报表已生成")')
                                        success_status.wait_for(timeout=60000)  # 等待最多 60 秒

                                        # 找到这一行里的下载按钮并点击
                                        download_button = report_row.locator('a:has-text("下载")')

                                        # 监听下载事件
                                        with page.expect_download() as download_info:
                                            download_button.click()

                                        # 获取下载的压缩文件对象
                                        download = download_info.value
                                        suggested_filename = download.suggested_filename
                                        # 保存下载的文件到指定路径
                                        download.save_as(os.path.join(download_folder, suggested_filename))
                                        df_jdqzyx = self.get_file_to_pd(download_folder, suggested_filename)
                                        df_jdqzyx = df_jdqzyx.copy()
                                        # df_jdqzyx = df_jdqzyx.iloc[[0]].reset_index(drop=True)
                                        df_jdqzyx = df_jdqzyx.reset_index(drop=True)
                                        df_jdqzyx = df_jdqzyx.loc[:, ~df_jdqzyx.columns.str.contains('当前时间')]
                                        df_jdqzyx = df_jdqzyx.loc[:, ~df_jdqzyx.columns.str.contains('全站订单成本')]

                                        # 设置新的列名
                                        df_jdqzyx.columns = [
                                            '点击时间', '花费', '投产比', '总订单金额', '总订单行', '展现数', '点击数'
                                        ]
                                        df_jdqzyx['点击时间'] = to_datetime(df_jdqzyx['点击时间'],
                                                                            format='%Y%m%d').dt.strftime(
                                            '%Y-%m-%d 00:00:00')
                                        df_jdqzyx.insert(0, '产品线', '全站营销')
                                        df_jdqzyx.insert(1, '产品线描述', '全站营销')
                                        df_jdqzyx.columns = ['scene_id', 'scene_name', 'date', 'cost', 'roi', 'total_sales_amount','total_sales_orders', 'impressions',  'clicks']
                                        # 重新索引 df_jdqzyx 的每一行，使其与 df_jdyxcjr 的列结构一致
                                        new_rows = df_jdqzyx.reindex(columns=df_jdyxcjr.columns)
                                        # 将新行追加到 df_jdyxcjr 中
                                        df_jdyxcjr = concat([df_jdyxcjr, new_rows], ignore_index=True)

                                        del df_jdqzyx

                                    if not df_jdyxcjr.empty:
                                        threading.Thread(target=auto_data_to_mysql, args=(df_jdyxcjr, 'ecapp_yxcj', store, dburl, None, store_type[store], None, today,task_status)).start()
                                        task_status[store]['YXCJ'] = True
                                    else:
                                        task_status[store]['YXCJ'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--营销场景-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      4. 京东-全商品核心概况-日      =========================#
                        self.status_signal.emit("处理全商品核心概况...")
                        jdhxgkr_flag = True  # 追踪的京东商品的HXGKR表是否全是True,全是True返回True,否则重新下载一次全商品核心概况
                        for proId in product_list:
                            if not task_status[store][proId]['HXGKR']:
                                jdhxgkr_flag = False
                                break
                        if not jdhxgkr_flag and store == '青岛昌隆文具有限公司':
                            try:
                                jdhxgkr_url = get_setting('TASK_URL')['JDHXGKR']
                                page.goto(jdhxgkr_url)
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                self.remove_jd_ad(page)
                                # 获取所有包含 '经营状况' 的 div 元素
                                target_divs = page.locator('div.list-title-text:has(p:has-text("经营状况"))')
                                count = target_divs.count()
                                for i in range(count):
                                    sub_title = target_divs.nth(i).locator('p.list-subTitle').text_content()
                                    # 判断是否是 '商品明细报表'
                                    if sub_title == '商品明细报表':
                                        # 监听打开新页面事件
                                        with page.expect_popup() as new_page:
                                            target_divs.nth(i).locator('p.list-subTitle').click()
                                        break
                                page.wait_for_load_state('load')
                                time.sleep(random.randint(self.min_wait, self.max_wait))

                                down_page = new_page.value
                                down_page.wait_for_load_state()
                                download_button = down_page.locator('span.download-data:has-text("下载报表")')
                                download_button.click()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 获取弹窗中的文本,使用正则表达式提取文件名
                                modal_text = down_page.locator('div.confirm-warning-text').text_content()
                                file_name = re.search(r'【(.*?)】', modal_text).group(1)
                                file_name = file_name.replace("商品明细报表", "商品明细日报")
                                # 替换日期部分
                                file_name = re.sub(r'(\d{4}-\d{2}-\d{2})-\d{4}-\d{2}-\d{2}', r'\1', file_name)
                                # 替换汇总展示为汇总-SKU
                                file_name = file_name.replace("汇总展示", "汇总-SKU")
                                # 添加文件后缀
                                jdhxgkr_file_name_value = file_name + '.xlsx'
                                jdreport_url = get_setting('TASK_URL')['JDREPORT']
                                down_page.goto(jdreport_url)
                                down_page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))

                                report_row = down_page.locator(
                                    f'tr:has(div.reportName:text("{jdhxgkr_file_name_value}"))').first
                                success_status = report_row.locator('td span.tag-status.success:has-text("已生成")')
                                success_status.wait_for(timeout=60000)  # 等待最多 60 秒

                                # 找到这一行里的下载按钮并点击
                                download_button = report_row.locator('td div.grid-operator-btn span:text("下载")')
                                # 监听下载事件
                                with down_page.expect_download() as download_info:
                                    download_button.click()

                                # 获取下载的压缩文件对象
                                download = download_info.value

                                # 获取下载文件的建议文件名
                                suggested_filename = download.suggested_filename
                                # 保存下载的文件到指定路径
                                download.save_as(os.path.join(download_folder, suggested_filename))
                                df_jdhxgkr = self.get_file_to_pd(download_folder, suggested_filename)
                                df_jdhxgkr = df_jdhxgkr.copy()
                                # df_jdhxgkr = df_jdhxgkr.drop(['品牌', '一级类目', '二级类目', '三级类目', '店铺名称'], axis=1)
                                columns_to_drop = ['品牌', '一级类目', '二级类目', '三级类目', '店铺名称']
                                existing_columns_to_drop = list(
                                    set(columns_to_drop) & set(df_jdhxgkr.columns))  # 获取存在的列
                                df_jdhxgkr = df_jdhxgkr.drop(existing_columns_to_drop, axis=1)

                                df_jdhxgkr['SKU'] = df_jdhxgkr['SKU'].astype(str).replace(r'\.0$', '', regex=True)
                                # 只保留需要跟踪的商品id的数据
                                # df_jdhxgkr = df_jdhxgkr[df_jdhxgkr['SKU'].isin(product_list)]==>此处去掉,因需要获取POP店铺的
                                df_jdhxgkr = df_jdhxgkr[df_jdhxgkr['SKU'].isin(product_list_bx)]

                                # 根据类型填充经营模式的数据
                                # 自营部分
                                df_jdhxgkr_self = df_jdhxgkr[df_jdhxgkr['经营模式'] == '自营']
                                df_jdhxgkr_self['时间'] = yesterday0
                                df_jdhxgkr_self.loc[df_jdhxgkr_self['加购商品件数'] < 0, '加购商品件数'] = 0
                                if not df_jdhxgkr_self.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_jdhxgkr_self, 'ecapp_hxgkr', store, dburl, None, store_type[store], product_list,
                                        today, task_status)).start()
                                for proId in product_list:
                                    task_status[store][proId]['HXGKR'] = True
                                # POP部分
                                df_jdhxgkr_pop = df_jdhxgkr[df_jdhxgkr['经营模式'] == 'POP']
                                df_jdhxgkr_pop['时间'] = yesterday0
                                df_jdhxgkr_pop.loc[df_jdhxgkr_pop['加购商品件数'] < 0, '加购商品件数'] = 0
                                if not df_jdhxgkr_pop.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(
                                        df_jdhxgkr_pop, 'ecapp_hxgkr', '白雪文具官方旗舰店', dburl, None, '京东',
                                        product_jd_list,
                                        today, task_status)).start()
                                for proId in product_jd_list:
                                    task_status['白雪文具官方旗舰店'][proId]['HXGKR'] = True

                                try:
                                    # 下载完后关闭新打开的页面
                                    down_page.close()
                                except:
                                    pass
                            except BaseException as e:
                                proMsg = '--' + store + '--全商品核心概况-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      5.京东-全商品推广数据表ZTR      =========================#
                        self.status_signal.emit("处理全商品推广数据...")
                        if not task_status[store]['ZTR_QZ']:
                            try:
                                if store == '白雪文具官方旗舰店':
                                    jdztr_url = get_setting('TASK_URL')['JDZTRNEW']
                                    page.goto(jdztr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    self.remove_jd_ad(page)
                                    # 移除弹窗
                                    self.remove_element(page, 'class', "jad-pc-modal-mask")
                                    date_input = page.locator("input.jad-pc-input[placeholder='请选择日期']")
                                    date_text = yesterday + '至' + yesterday
                                    date_input.fill(date_text)
                                    date_input.press("Enter")
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    download_btn = page.locator("div.download-report-wrap.w32", has=page.locator("i.jzticon-xiazai"))
                                    download_btn.click(timeout=5000)
                                    card = page.locator("div.download-card").first
                                    card.wait_for(state="visible", timeout=5000)  # 最多等待 5 秒
                                    input_box = card.locator("input.jad-pc-input")
                                    jdztr_file_name_value = input_box.input_value()
                                    start_btn = card.locator("button.jad-pc-btn-primary", has_text="开始生成")
                                    start_btn.click(timeout=5000)
                                    time.sleep(random.randint(7, 8))
                                else:
                                    jdztr_url = get_setting('TASK_URL')['JDZTR']
                                    page.goto(jdztr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    self.remove_jd_ad(page)
                                    # 移除弹窗
                                    self.remove_element(page, 'class', "jad-pc-modal-mask")
                                    max_retries = 3  # 最多重试 3 次
                                    # 点击日期,选择昨天
                                    for attempt in range(max_retries):
                                        try:
                                            # 1. 点击日期选择框按钮
                                            page.locator("button.jad-btn.jad-btn-default.jad-pro-date-compare-double-date-frame").click()

                                            # 2. 等待“昨天”选项可见
                                            yesterday_p = page.locator("label.jad-radio-wrapper >> text=昨天")
                                            yesterday_p.wait_for(state="visible", timeout=5000)  # 最多等待 5 秒

                                            # 3. 点击“昨天”
                                            yesterday_p.click(timeout=5000)
                                            max_retries = 3
                                            break  # 如果成功执行，跳出循环
                                        except Exception as e:
                                            if attempt == max_retries - 1:
                                                raise  # 3 次都失败，抛出异常
                                            time.sleep(2)
                                    # 1. 点击确认按钮
                                    confirm_button = page.locator("button.jad-btn.jad-btn-primary >> text=确认")
                                    confirm_button.wait_for(state="visible", timeout=5000)  # 等待按钮可见
                                    confirm_button.click(timeout=5000)
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    download_btn = page.locator('button[title="下载报表"]')
                                    download_btn.scroll_into_view_if_needed()
                                    download_btn.click(timeout=5000)
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 定位弹出的下载报表弹框
                                    tooltip = page.locator("div.jad-tooltip-inner.jad-tooltip-light-inner")

                                    # 获取 input 中默认显示的文件名
                                    report_input = tooltip.locator("input.jad-input")
                                    jdztr_file_name_value = report_input.input_value()  # 获取 input 显示的值

                                    # 点击“开始生成”按钮
                                    start_btn = tooltip.locator("button:has-text('开始生成')")
                                    with page.expect_popup() as popup_info:
                                        start_btn.click(timeout=10000)
                                    new_page = popup_info.value
                                    new_page.wait_for_load_state()

                                if jdztr_file_name_value != '':
                                    if store == '白雪文具官方旗舰店':
                                        report_row = page.locator(f'tr:has-text("{jdztr_file_name_value}")').first
                                    else:
                                        report_row = new_page.locator(f'tr:has-text("{jdztr_file_name_value}")').first
                                    success_status = report_row.locator('span.download-report-font-green:has-text("报表已生成")')
                                    success_status.wait_for(timeout=60000)  # 等待最多 60 秒

                                    # 找到这一行里的下载按钮并点击
                                    download_button = report_row.locator('a:has-text("下载")')

                                    if store == '白雪文具官方旗舰店':
                                        # 监听下载事件
                                        with page.expect_download() as download_info:
                                            download_button.click()
                                    else:
                                        with new_page.expect_download() as download_info:
                                            download_button.click()
                                    # 获取下载的压缩文件对象
                                    download = download_info.value

                                    # 获取下载文件的建议文件名
                                    suggested_filename = download.suggested_filename
                                    # 保存下载的文件到指定路径
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_jdztr = self.get_file_to_pd(download_folder, suggested_filename)
                                    df_jdztr = df_jdztr.copy()
                                    # 保留状态=有效的行
                                    df_jdztr = df_jdztr[df_jdztr['状态'] == '有效']

                                    # 删除实际存在的列
                                    # columns_to_drop = ['状态', '日预算', '时段', '折扣系数', '时效开始时间', '时效结束时间', '展现数', '总加购数', '加购率(%)', '直接订单行']
                                    # existing_columns_to_drop = list(set(columns_to_drop) & set(df_jdztr.columns))  # 获取存在的列
                                    # df_jdztr = df_jdztr.drop(existing_columns_to_drop, axis=1)
                                    df_jdztr.insert(0, '日期', [yesterday0] * len(df_jdztr))
                                    # 过滤掉没有/的数据,赋值给ID
                                    df_jdztr = df_jdztr[df_jdztr['推广计划'].str.contains('/', na=False)].copy()
                                    df_jdztr['ID'] = df_jdztr['推广计划'].str.split('/').str[0]
                                    df_jdztr['类别'] = '全站推广'
                                    df_jdztr['状态'] = '商品'

                                    columns = ['日期', '推广计划', 'ID', '状态', '点击率(%)', '花费',
                                               '平均点击成本', '转化率(%)', '投产比', '总订单金额', '总订单行',
                                               '点击数', '类别', '展现数']
                                    df_jdztr = df_jdztr[columns]
                                    key = '全站推广'
                                    tableName = 'ZTR_QZ'
                                    task_status[store]['ZTR_GJ'] = True
                                    task_status[store]['ZTR_RQ'] = True
                                    task_status[store]['ZTR_HP'] = True
                                    if store != '白雪文具官方旗舰店':
                                        new_page.close()
                                    if not df_jdztr.empty:
                                        threading.Thread(target=auto_data_to_mysql, args=(
                                            df_jdztr, 'ecapp_ztr', store, dburl, key, store_type[store], None, today,
                                            task_status)).start()
                                        task_status[store][tableName] = True
                                    else:
                                        task_status[store][tableName] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                            except BaseException as e:
                                proMsg = '--' + store + '--全商品推广-普通计划获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                    elif store_type[store] == '拼多多':
                        proMsg = '--' + store + '开始检查/获取店铺级数据'
                        self.logger.info(proMsg)
                        task_status[store]['LLLYDPR'] = True
                        task_status[store]['ZTR_GJ'] = True
                        task_status[store]['ZTR_RQ'] = True
                        task_status[store]['ZTR_HP'] = True
                        # =========================      2.数据总览-日      =========================#
                        self.status_signal.emit("处理数据总览...")
                        if not task_status[store]['SJZLR']:
                            try:
                                url = get_setting('TASK_URL')['PDDSJZLR']
                                page.goto(url)
                                # 等待页面加载
                                page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 检查登录数据是否过期
                                login_btn = page.locator('//button[span[text()="登录"]]')
                                if login_btn.count() > 0:
                                    proMsg = '--' + store + '登录数据已过期,请重新保存登录数据后再重新下载本店铺数据!'
                                    self.logger.error(proMsg)
                                    try:
                                        context.close()
                                    except:
                                        pass
                                    try:
                                        browser.close()
                                    except:
                                        pass
                                    continue
                                # 侦测弹框并取消
                                page.on("dialog", lambda dialog: dialog.dismiss())
                                # 检测弹窗是否存在
                                try:
                                    # 定位弹窗的关闭按钮
                                    close_button = page.query_selector("svg[data-testid='beast-core-icon-close']")
                                    if close_button:
                                        close_button.click()  # 点击关闭
                                    # 移除浏览器检测弹窗
                                    self.remove_element(page, 'id', "global_broswer_nav")
                                except Exception as e:
                                    pass
                                page.keyboard.press('Escape')

                                # # 获得基准字体,当前页面字体,基准字体与数字映射关系
                                # base_font,pdd_font,base_mapping = self.base_mapping(page)
                                # # 生成拼多多字体的映射
                                # pdd_to_num_map = self.generate_pdd_mapping(base_font,pdd_font,base_mapping,)
                                #
                                # # 使用 JavaScript 获取正常文本
                                # values = page.eval_on_selector_all(
                                #     "div.newCardInnerWrapper .value.__spider_font",
                                #     "elements => elements.map(el => el.innerText)"
                                # )
                                # # 替换加密字符
                                # result = self.replace_encrypted_numbers(values, pdd_to_num_map)
                                if exe_path is None:
                                    raise ValueError('未找到视觉识别依赖库安装路径,请点击[下载依赖库]按钮进行下载安装!')
                                # 创建截图目录
                                img_path = os.path.join(os.getcwd(), get_setting('TESSERACT')['TESSERACT_IMG'])
                                os.makedirs(img_path, exist_ok=True)
                                page.wait_for_selector(".mms-symc-card-item-new__wrapper")
                                time.sleep(random.randint(3, 5))
                                # 获取所有卡片
                                cards = page.locator(".mms-symc-card-item-new__wrapper").all()
                                sjzlr_dict = {'成交金额': '', '成交买家数': '','成交订单数': '', '成交转化率': '', '客单价': '',
                                              '店铺访客数': '', '店铺浏览量': ''}
                                for index, card in enumerate(cards, 1):
                                    try:
                                        # 获取指标名称
                                        alias_element = card.locator(".alias").first
                                        alias = alias_element.inner_text()
                                        safe_name = self.get_filename(alias) or f"unnamed_{index}"
                                        if safe_name in sjzlr_dict:
                                            # 定位目标元素
                                            target_element = card.locator(
                                                "p.valueWrapper >> span.value.__spider_font").first
                                            target_element.scroll_into_view_if_needed()

                                            # 截图配置
                                            screen_path = f"{img_path}/{safe_name}.png"
                                            target_element.screenshot(
                                                path=screen_path,
                                                timeout=5000,
                                                animations="disabled",
                                                caret="initial"
                                            )
                                            sjzlr_dict[safe_name] = self.ocr_img_num(screen_path, exe_path)
                                    except Exception as e:
                                        self.logger.error(f"拼多多数据总览图像识别失败{str(e)}")

                                df = DataFrame([sjzlr_dict])
                                df = df.apply(to_numeric, errors='ignore')  # 转换为数值类型，忽略无法转换的值
                                #  ================================获取客服数据
                                self.status_signal.emit("处理客服数据...")
                                kf_dict = {'咨询人数': 0, '咨询率': '0%', '平均人工响应时长': 0, '成功退款金额': 0}

                                try:
                                    kfr_url = get_setting('TASK_URL')['PDDKFR']
                                    page.goto(kfr_url)
                                    ondDayBtn = page.locator('button:has-text("近1天")').first
                                    ondDayBtn.click(force=True, timeout=5000)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 获取咨询人数(人)
                                    kf_dict['咨询人数'] = page.query_selector("text='咨询人数(人)'").query_selector(
                                        "xpath=../../following-sibling::section[1]/div/div").text_content()
                                    # 获取3分钟人工回复率
                                    kf_dict['咨询率'] = page.query_selector("text='3分钟人工回复率'").query_selector(
                                        "xpath=../../following-sibling::section[1]/div/div").text_content()
                                    # 获取平均人工响应时长
                                    kf_dict['平均人工响应时长'] = page.query_selector(
                                        "text='平均人工响应时长'").query_selector(
                                        "xpath=../../following-sibling::section[1]/div/div").text_content()
                                except BaseException as e:
                                    proMsg = '--' + store + '--数据总览中的客服数据获取失败' + str(e)
                                    self.logger.warning(proMsg)
                                #  ================================获取退款数据
                                self.status_signal.emit("处理退款数据...")
                                try:
                                    tkr_url = get_setting('TASK_URL')['PDDTKR']
                                    page.goto(tkr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    refund_amount_parent = page.locator(
                                        "div[data-tracking-click-viewid='refund_amount']")
                                    refund_amount_parent.scroll_into_view_if_needed()
                                    # 定位到退款金额的值
                                    refund_amount_element = refund_amount_parent.locator("span.__spider_font").first
                                    screen_path = f"{img_path}/退款金额.png"
                                    refund_amount_element.screenshot(path=screen_path, timeout=5000,
                                                                     animations="disabled", caret="initial")
                                    kf_dict['成功退款金额'] = self.ocr_img_num(screen_path, exe_path)
                                except BaseException as e:
                                    proMsg = '--' + store + '--数据总览中的退款数据获取失败' + str(e)
                                    self.logger.warning(proMsg)
                                kf_df = DataFrame([kf_dict])
                                kf_df = kf_df.apply(to_numeric, errors='ignore')
                                verbose_name_list = [
                                    '日期', '类别', '成交金额', '店铺客户数', '平均停留时长', '成交买家数',
                                    '成交转化率', '客单价',
                                    '老客复购率', '老客复购人数', '老客复购金额', '成交订单数', '支付件数',
                                    '店铺访客数', '店铺浏览量',
                                    '成功退款金额', '净支付金额', '加购件数', '加购人数', '商品收藏人数', '咨询率',
                                    '平均人工响应时长',
                                    '24小时揽收及时率', '物流到货时长(小时)', '退款处理时长(天)', '成功退款率',
                                    '平台判责率', '全站推广花费',
                                    '关键词推广花费', '精准人群推广花费', '智能场景花费', '淘宝客佣金', '支付老买家数',
                                    '回访成交', '咨询人数',
                                    '内容查看人数', '有过已购状态的客户数', '旺旺满意度', '总支付金额',
                                    '48小时揽收及时率', '关联商品数',
                                    '总内容供给规模量', '纠纷投诉商责率', '商品覆盖率', '开播时长(小时)', '询单转化率',
                                    '老买家支付金额',
                                    '进入公域视频数', '总支付子订单数', '停留时长'
                                ]
                                common_columns = ['成交金额', '成交买家数','成交订单数', '成交转化率', '客单价', '店铺访客数',
                                                  '店铺浏览量']
                                common_columns1 = ['咨询人数', '咨询率', '平均人工响应时长', '成功退款金额']
                                # 创建空的 DataFrame
                                df_sjzlr = DataFrame(columns=verbose_name_list)
                                df_sjzlr[common_columns] = df[common_columns]
                                df_sjzlr[common_columns1] = kf_df[common_columns1]
                                df_sjzlr['日期'] = yesterday0
                                df_sjzlr['类别'] = '本店'
                                df_sjzlr['总支付金额'] = df_sjzlr["成交金额"]
                                if not df_sjzlr.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(df_sjzlr, 'ecapp_sjzlr', store, dburl, None, None, None, today,task_status)).start()
                                    task_status[store]['SJZLR'] = True
                                else:
                                    task_status[store]['SJZLR'] = True
                                    threading.Thread(target=only_up_task_status,
                                                     args=(dburl, today, task_status)).start()
                                    self.logger.warning('--' + store + '--数据总览-日数据为空')
                                store_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--数据总览-日获取失败' + str(e)
                                self.logger.warning(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        # =========================      1.营销场景-日      =========================#
                        self.status_signal.emit("处理营销场景...")
                        if not task_status[store]['YXCJ']:
                            try:
                                # pddyxcj_url = get_setting('TASK_URL')['PDDYXCJ'] + yesterday + '&endDate=' + yesterday
                                # page.goto(pddyxcj_url)
                                # page.wait_for_load_state()
                                # time.sleep(random.randint(self.min_wait, self.max_wait))
                                page_title = page.title()
                                if page_title != '交易概况':
                                    tkr_url = get_setting('TASK_URL')['PDDTKR']
                                    page.goto(tkr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                promotion_link = page.query_selector("a:has-text('推广报表')")
                                # 点击链接并监听新页面打开
                                with page.expect_popup() as popup_info:
                                    promotion_link.click()
                                new_page = popup_info.value
                                new_page.wait_for_load_state()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                # 检测弹窗是否存在
                                try:
                                    # 定位弹窗的关闭按钮
                                    close_button = new_page.query_selector("svg[data-testid='beast-core-icon-close']")
                                    if close_button:
                                        close_button.click()  # 点击关闭
                                except Exception as e:
                                    pass

                                button_locator = new_page.locator('span:has-text("表格")')
                                button_locator.click()
                                new_page.wait_for_load_state()
                                yesterday_str = yesterday.replace("-", "/")  # 格式化为 YYYY/MM/DD
                                start_date_input = new_page.locator("input[placeholder='开始日期']").first
                                start_date_input.click()
                                start_date_input.fill(yesterday_str)
                                end_date_input = new_page.locator("input[placeholder='结束日期']").first
                                end_date_input.click()
                                end_date_input.fill(yesterday_str)
                                end_date_input.press('Enter')
                                new_page.wait_for_timeout(3000)

                                tfoot = new_page.query_selector(".anq-table-summary")

                                # 提取数据
                                total_cost = tfoot.query_selector("tr > td:nth-child(2) > div > div:nth-child(1)").text_content().replace(",", "")
                                total_sales = tfoot.query_selector("tr > td:nth-child(4) > div > div:nth-child(1)").text_content().replace(",", "")
                                actual_roi = tfoot.query_selector("tr > td:nth-child(5) > div > div:nth-child(1)").text_content().replace(",", "")
                                # impressions = tfoot.query_selector("tr > td:nth-child(25) > div > div:nth-child(1)").text_content().replace(",", "")
                                # clicks = tfoot.query_selector("tr > td:nth-child(26) > div > div:nth-child(1)").text_content().replace(",", "") 拼多多不同店铺之间曝光量和点击量位置不同,改为取最后两列
                                tds = tfoot.query_selector_all("td")
                                impressions = tds[-2].query_selector("div > div").text_content().replace(",", "")
                                clicks = tds[-1].query_selector("div > div").text_content().replace(",", "")
                                avg_click_cost = round(float(total_sales) / float(clicks), 2) if float(clicks) > 0 else 0

                                def set_data(yesterday0, yxcj_type, impressions, clicks, total_cost, avg_click_cost,
                                             total_sales, actual_roi):
                                    data = {
                                        "日期": [yesterday0], "场景ID": [yxcj_type], "场景名字": [yxcj_type],
                                        "展现量": [int(impressions)], "点击量": [int(clicks)],
                                        "花费": [float(total_cost)],
                                        "点击率": [""], "平均点击花费": [avg_click_cost], "千次展现花费": [""],
                                        "总预售成交金额": [""], "总预售成交笔数": [""], "直接预售成交金额": [""],
                                        "直接预售成交笔数": [""], "间接预售成交金额": [""], "间接预售成交笔数": [""],
                                        "直接成交金额": [""], "间接成交金额": [""], "总成交金额": [float(total_sales)],
                                        "总成交笔数": [""], "直接成交笔数": [""], "间接成交笔数": [""],
                                        "点击转化率": [""], "投入产出比": [float(actual_roi)], "总成交成本": [""],
                                        "总购物车数": [""], "直接购物车数": [""], "间接购物车数": [""], "加购率": [""],
                                        "收藏宝贝数": [""], "收藏店铺数": [""], "店铺收藏成本": [""],
                                        "总收藏加购数": [""], "总收藏加购成本": [""], "宝贝收藏加购数": [""],
                                        "宝贝收藏加购成本": [""], "总收藏数": [""], "宝贝收藏成本": [""],
                                        "宝贝收藏率": [""], "加购成本": [""], "拍下订单笔数": [""],
                                        "拍下订单金额": [""], "直接收藏宝贝数": [""], "间接收藏宝贝数": [""],
                                        "优惠券领取量": [""], "购物金充值笔数": [""], "购物金充值金额": [""],
                                        "旺旺咨询量": [""], "引导访问量": [""], "引导访问人数": [""],
                                        "引导访问潜客数": [""], "引导访问潜客占比": [""], "入会率": [""],
                                        "入会量": [""], "引导访问率": [""], "深度访问量": [""], "平均访问页面数": [""],
                                        "成交新客数": [""], "成交新客占比": [""], "会员首购人数": [""],
                                        "会员成交金额": [""], "会员成交笔数": [""], "成交人数": [""],
                                        "人均成交笔数": [""], "人均成交金额": [""]
                                    }
                                    return data

                                data_1 = set_data(yesterday0, "全站推广", impressions, clicks, total_cost,avg_click_cost, total_sales, actual_roi)
                                df1 = DataFrame(data_1)
                                data_2 = {}
                                df2 = None
                                # df_pddyxcj = DataFrame(data)

                                # 增加取明星店铺数据
                                starsBtn = page.locator("span:has-text('明星店铺')").first
                                if starsBtn.count() > 0:
                                    # 点击链接并监听新页面打开
                                    with page.expect_popup() as popup_info:
                                        starsBtn.click()
                                    page_3 = popup_info.value
                                    page_3.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    reportBtn = page_3.locator('span:has-text("查看报表")')
                                    if reportBtn.count() > 0:
                                        yxcj2_url = get_setting('TASK_URL')['PDDYXCJ2']
                                        yxcj2_url = yxcj2_url + yesterday + '&endDate=' + yesterday
                                        page_3.goto(yxcj2_url)
                                        page_3.wait_for_load_state()
                                        time.sleep(random.randint(self.min_wait, self.max_wait))
                                        button_locator = page_3.locator('span:has-text("表格")')
                                        button_locator.click()
                                        page_3.wait_for_load_state()

                                        tfoot = page_3.query_selector(".anq-table-summary")

                                        # 提取数据
                                        total_cost_2 = tfoot.query_selector(
                                            "tr > td:nth-child(2) > div > div:nth-child(1)").text_content().replace(",",
                                                                                                                    "")
                                        total_sales_2 = tfoot.query_selector(
                                            "tr > td:nth-child(3) > div > div:nth-child(1)").text_content().replace(",",
                                                                                                                    "")
                                        actual_roi_2 = tfoot.query_selector(
                                            "tr > td:nth-child(4) > div > div:nth-child(1)").text_content().replace(",",
                                                                                                                    "")
                                        impressions_2 = tfoot.query_selector(
                                            "tr > td:nth-child(8) > div > div:nth-child(1)").text_content().replace(",",
                                                                                                                    "")
                                        clicks_2 = tfoot.query_selector(
                                            "tr > td:nth-child(9) > div > div:nth-child(1)").text_content().replace(",",
                                                                                                                    "")
                                        try:
                                            avg_click_cost_2 = round(float(total_sales_2) / float(clicks_2), 2) if float(clicks) > 0 else 0
                                        except:
                                            avg_click_cost_2 = 0
                                        data_2 = set_data(yesterday0, "明星店铺", impressions_2, clicks_2, total_cost_2,avg_click_cost_2, total_sales_2, actual_roi_2)
                                        df2 = DataFrame(data_2)
                                        page_3.close()
                                    else:
                                        page_3.close()
                                if df2 is not None:
                                    df_pddyxcj = concat([df1, df2], ignore_index=True)
                                else:
                                    df_pddyxcj = df1
                                # =========================      增加多多进宝      =========================#
                                self.status_signal.emit("处理多多进宝...")
                                try:
                                    yxcj_ddjbr = get_setting('TASK_URL')['PDDDDJBR']
                                    new_page.goto(yxcj_ddjbr)
                                    new_page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    new_page.evaluate("""
                                                         () => {
                                                                const element = document.getElementById('umd_kits_home_entry');
                                                                      if (element) {
                                                                          element.remove();
                                                                      }
                                                                }
                                                     """)
                                    new_page.locator('div[data-testid="beast-core-select"]').last.click(timeout=3000)
                                    new_page.wait_for_load_state()
                                    new_page.locator('li[data-checked="false"] >> text=昨天').click(timeout=3000)
                                    new_page.locator('button >> text="查询"').click()
                                    new_page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 获取成交笔数
                                    # trade_count = page.query_selector('div:has-text("成交笔数") + div').text_content()
                                    # 获取成交金额
                                    trade_amount = new_page.query_selector('div:has-text("成交金额（元）") + div').text_content()
                                    # 获取预估支付佣金
                                    commission = new_page.query_selector('div:has-text("预估支付佣金（元）") + div').text_content()
                                    data_3 = set_data(yesterday0, "多多进宝", 0, 0, commission, 0.0, trade_amount, 0.0)
                                    df3 = DataFrame(data_3)
                                    if df3 is not None:
                                        df_pddyxcj = concat([df_pddyxcj, df3], ignore_index=True)
                                except BaseException as e:
                                    pass
                                if not df_pddyxcj.empty:
                                    threading.Thread(target=auto_data_to_mysql, args=(df_pddyxcj, 'ecapp_yxcj', store, dburl, None, None, None, today,
                                        task_status)).start()
                                    task_status[store]['YXCJ'] = True
                                    # task_status[store]['JH'] = True
                                else:
                                    task_status[store]['YXCJ'] = True
                                    # task_status[store]['JH'] = True
                                    threading.Thread(target=only_up_task_status,args=(dburl, today, task_status)).start()
                                # =========================      3.拼多多-全商品推广数据表ZTR  放在同一页面处理,减少跳转    =========================#
                                self.status_signal.emit("处理商品全站推广数据...")
                                if not task_status[store]['ZTR_QZ']:
                                    try:
                                        pddztr_url = get_setting('TASK_URL')['PDDZTR']
                                        pddztr_url = pddztr_url + yesterday + '&endDate=' + yesterday
                                        new_page.goto(pddztr_url)
                                        new_page.wait_for_load_state()
                                        time.sleep(random.randint(self.min_wait, self.max_wait))

                                        download_button = new_page.locator("button:has(span:has-text('下载'))").last
                                        # 监听下载事件
                                        with new_page.expect_download() as download_info:
                                            download_button.click()

                                        download = download_info.value
                                        suggested_filename = download.suggested_filename
                                        # 保存下载的文件到指定路径
                                        download.save_as(os.path.join(download_folder, suggested_filename))
                                        down_df_pddztr = self.get_file_to_pd(download_folder, suggested_filename)
                                        # 筛选商品ID在 product_list 中的数据
                                        filtered_df = down_df_pddztr[down_df_pddztr['商品ID'].isin(product_list)]
                                        verbose_name_list = [
                                            '日期', '商品ID', '主体类型', '主体名称', '曝光量', '点击量',
                                            '成交花费(元)', '点击率', '平均点击花费',
                                            '千次展现花费', '总预售成交金额', '总预售成交笔数', '直接预售成交金额',
                                            '直接预售成交笔数',
                                            '间接预售成交金额', '间接预售成交笔数', '直接成交金额', '净交易额(元)',
                                            '交易额(元)',
                                            '成交笔数', '净成交笔数', '间接成交笔数', '点击转化率', '实际投产比',
                                            '总花费(元)',
                                            '总购物车数', '直接购物车数', '间接购物车数', '加购率', '收藏宝贝数',
                                            '收藏店铺数',
                                            '店铺收藏成本', '总收藏加购数', '总收藏加购成本', '宝贝收藏加购数',
                                            '宝贝收藏加购成本',
                                            '总收藏数', '宝贝收藏成本', '宝贝收藏率', '加购成本', '拍下订单笔数',
                                            '拍下订单金额',
                                            '直接收藏宝贝数', '间接收藏宝贝数', '优惠券领取量', '购物金充值笔数',
                                            '购物金充值金额',
                                            '旺旺咨询量', '引导访问量', '引导访问人数', '引导访问潜客数',
                                            '引导访问潜客占比',
                                            '入会率', '入会量', '引导访问率', '深度访问量', '平均访问页面数',
                                            '成交新客数',
                                            '成交新客占比', '会员首购人数', '会员成交金额', '会员成交笔数', '成交人数',
                                            '人均成交笔数', '人均成交金额', '自然流量转化金额', '自然流量曝光量', '类别'
                                        ]
                                        common_columns = ['商品ID', '总花费(元)', '成交花费(元)', '交易额(元)',
                                                          '实际投产比', '净交易额(元)', '成交笔数', '曝光量', '点击量']
                                        df_pddztr = DataFrame(columns=verbose_name_list)
                                        df_pddztr[common_columns] = filtered_df[common_columns]
                                        df_pddztr['日期'] = yesterday0
                                        df_pddztr['主体类型'] = '商品'
                                        # df_pddztr['点击率'] = np.where(df_pddztr['曝光量'] != 0,df_pddztr['点击量'] / df_pddztr['曝光量'], 0)  避免除0问题  20250821
                                        # df_pddztr['平均点击花费'] = np.where(df_pddztr['点击量'] != 0,df_pddztr['成交花费(元)'] / df_pddztr['点击量'], 0)
                                        ddjb_clicks = df_pddztr['点击量'].astype(float)
                                        ddjb_impressions = df_pddztr['曝光量'].astype(float)
                                        ddjb_cost = df_pddztr['成交花费(元)'].astype(float)
                                        df_pddztr['点击率'] = np.divide(ddjb_clicks, ddjb_impressions,
                                                                        out=np.zeros_like(ddjb_clicks),
                                                                        where=ddjb_impressions != 0)
                                        df_pddztr['平均点击花费'] = np.divide(ddjb_cost, ddjb_clicks,
                                                                           out=np.zeros_like(ddjb_cost),
                                                                           where=ddjb_clicks != 0)
                                        if not df_pddztr.empty:
                                            key = '全站推广'
                                            threading.Thread(target=auto_data_to_mysql, args=(df_pddztr, 'ecapp_ztr', store, dburl, key, store_type[store], None, today,task_status)).start()
                                            task_status[store]['ZTR_QZ'] = True
                                        else:
                                            task_status[store]['ZTR_QZ'] = True
                                            threading.Thread(target=only_up_task_status,
                                                             args=(dburl, today, task_status)).start()
                                            self.logger.warning('--' + store + '--商品推广-全站推广数据为空')
                                    except BaseException as e:
                                        proMsg = '--' + store + '--商品推广-全站推广获取失败' + str(e)
                                        self.logger.warning(proMsg)
                                # 关闭新页面
                                new_page.close()
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                                store_flag = True
                            except BaseException as e:
                                proMsg = '--' + store + '--营销场景数据获取失败:' + str(e)
                                self.logger.error(proMsg)
                            time.sleep(random.randint(self.min_wait, self.max_wait))

                    proMsg = '--' + store + '开始检查/获取商品级数据'
                    self.logger.info(proMsg)
                    # 开始获取商品ID的数据
                    for zindex, product in enumerate(product_list):
                        zCount = len(product_list)
                        curr = zindex + 1  # 用来显示进度 curr/zCount
                        pro_flag = False  # 恢复默认值
                        if store_type[store] == '天猫':
                            # =========================      6.核心概况-日==>转到全商品下载,此处不再使用  2025-04-03      =========================#
                            # self.status_signal.emit(f"处理商品核心概况:{curr}/{zCount}")
                            # if not task_status[store][product]['HXGKR']:
                            #     try:
                            #         hxgkr_url = get_setting('TASK_URL')['HXGK'] + today + '%7C' + today + '&dateType=day&itemId='
                            #         hxgkr_url += product
                            #         page.goto(hxgkr_url)
                            #         page.wait_for_load_state()
                            #         time.sleep(random.randint(5, 8))
                            #         time.sleep(random.randint(self.min_wait, self.max_wait))
                            #
                            #         # 找到包含“核心概况”文本的 h4 元素   #不再使用店透视
                            #         core_overview_h4 = page.locator("h4.oui-card-title", has_text="核心概况")
                            #         # 获取 h4 所在的父级 div
                            #         parent_div = core_overview_h4.locator("xpath=..")  # `..` 代表父级元素
                            #         # 在这个父级 div 内找到下载按钮
                            #         export_option = parent_div.locator("a.sycm-traced-download-text")
                            #         export_option.scroll_into_view_if_needed()
                            #         # 处理可能的遮挡元素
                            #         page.evaluate("document.querySelector('#DTS_ENTRY__sycm')?.remove()")  # 移除拦截的元素
                            #         # 先 hover 再点击
                            #         export_option.hover()
                            #         # 监听下载事件
                            #         with page.expect_download() as download_info:
                            #             export_option.click()
                            #
                            #         download = download_info.value
                            #         suggested_filename = download.suggested_filename
                            #
                            #         # 保存下载的文件到指定路径
                            #         download.save_as(os.path.join(download_folder, suggested_filename))
                            #         df_hxgkr = self.get_file_to_pd(download_folder, suggested_filename)
                            #         df_hxgkr = df_hxgkr.drop(index=[0, 1, 2, 3])
                            #         new_columns = df_hxgkr.iloc[0]
                            #         df_hxgkr = df_hxgkr[1:]  # 删除
                            #         df_hxgkr.columns = new_columns
                            #         df_hxgkr = df_hxgkr.tail(1)  # 只保留最后一行
                            #
                            #         columns_to_keep = [
                            #             '时间', '商品访客数', '商品浏览量', '商品平均停留时长', '商品详情页跳出率', '商品加购件数', '商品加购人数',
                            #             '商品收藏人数', '下单买家数', '下单件数', '下单金额', '下单转换率', '支付买家数', '支付件数', '支付金额',
                            #             '支付转换率', '聚划算支付金额', '支付新买家数', '支付老买家数', '老买家支付金额', '成功退款金额'
                            #         ]
                            #
                            #         # 仅保留指定的字段并按顺序排列
                            #         # df_hxgkr = df_hxgkr[columns_to_keep]
                            #         df_hxgkr = df_hxgkr.reindex(columns=columns_to_keep, fill_value=0)#仅保留指定的字段并按顺序排列,不存在的列设置0
                            #         # 将时间列格式化为指定格式
                            #         df_hxgkr['时间'] = yesterday0
                            #         df_hxgkr.insert(0, '类别', '本品')
                            #         df_hxgkr.insert(0, '商品图片', '')
                            #         df_hxgkr.insert(0, '商品ID', product)
                            #         df_hxgkr.insert(0, '商品名称', '')
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('商品访客数') + 1, '商品微详情访客数', 0)
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('商品收藏人数') + 1, '访问加购转化率', '0%')
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('访问加购转化率') + 1, '访问收藏转化率', '0%')
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('老买家支付金额') + 1, 'UV价值', 0)
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('UV价值') + 1, '人均浏览量', 0)
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('人均浏览量') + 1, '详情页跳出人数', 0)
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('详情页跳出人数') + 1, '聚划算支付金额占比', '0%')
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('聚划算支付金额占比') + 1, '件单价', 0)
                            #         df_hxgkr.insert(df_hxgkr.columns.get_loc('件单价') + 1, '客单价', 0)
                            #
                            #         # page.keyboard.press('Escape')
                            #         if not df_hxgkr.empty:
                            #             threading.Thread(target=auto_data_to_mysql,
                            #                              args=(df_hxgkr, 'ecapp_hxgkr', store, dburl, None, None, product, today, task_status)).start()
                            #             task_status[store][product]['HXGKR'] = True
                            #         else:
                            #             task_status[store][product]['HXGKR'] = True
                            #             threading.Thread(target=only_up_task_status, args=(dburl, today, task_status)).start()
                            #         pro_flag = True
                            #     except BaseException as e:
                            #         proMsg = '--' + store + '--' + product + '--核心概况-日获取失败:' + str(e)
                            #         self.logger.warning(proMsg)
                            #     time.sleep(random.randint(self.min_wait, self.max_wait))
                            # =========================      7.流量来源-日      =========================#
                            self.status_signal.emit(f"处理商品流量来源:{curr}/{zCount}")
                            if not task_status[store][product]['LLLYR']:
                                try:
                                    lllyr_url = get_setting('TASK_URL')[
                                                    'LLLY'] + today + '%7C' + today + '&dateType=day&itemId='
                                    lllyr_url += product
                                    page.goto(lllyr_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    parent_span = page.locator(
                                        'img[src="https://img.alicdn.com/imgextra/i2/O1CN01AzNiUN1ErXLBM8y21_!!6000000000405-55-tps-12-12.svg"]').locator(
                                        '..')
                                    if parent_span.inner_text() == "返回旧版":
                                        parent_span.click()
                                    else:
                                        pass
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(5, 8))
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # button_selector = 'div#Dts_CcItemArchivesFlow button.el-button--success'
                                    # page.wait_for_selector(button_selector, timeout=5000)
                                    # page.click(button_selector)
                                    # time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # max_retries_lllyr = 3
                                    # zcount_lllyr = 0
                                    # while zcount_lllyr < max_retries_lllyr:
                                    #     try:
                                    #         page.wait_for_selector('.el-dialog__wrapper', timeout=5000)
                                    #         break
                                    #     except BaseException as e:
                                    #         zcount_lllyr += 1
                                    #         page.goto(lllyr_url)
                                    #         page.wait_for_load_state()
                                    #         page.click(button_selector)
                                    #         pass
                                    # time.sleep(random.randint(self.min_wait, self.max_wait))
                                    #
                                    # export_button = page.get_by_role("button", name="导出表格")
                                    # export_button_div = export_button.locator('..')
                                    # dropdown_button = export_button_div.locator('i.el-icon-arrow-down')
                                    # dropdown_button.scroll_into_view_if_needed()
                                    # dropdown_button.click()
                                    # export_option = page.locator('li:has-text("导出表格 csv(所有列)")')
                                    #
                                    # # 监听下载事件
                                    # with page.expect_download() as download_info:
                                    #     export_option.click()
                                    page.keyboard.press('Escape')
                                    export_option = page.locator("a.sycm-traced-download-text")
                                    export_option.scroll_into_view_if_needed()
                                    # 处理可能的遮挡元素
                                    page.evaluate("document.querySelector('#DTS_ENTRY__sycm')?.remove()")  # 移除拦截的元素
                                    # 先 hover 再点击
                                    export_option.hover()
                                    # 监听下载事件
                                    with page.expect_download() as download_info:
                                        export_option.click(force=True)  # 强制点击，绕过遮挡

                                    download = download_info.value
                                    suggested_filename = download.suggested_filename

                                    # 保存下载的文件到指定路径
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_lllyr = self.get_file_to_pd(download_folder, suggested_filename)

                                    df_lllyr = df_lllyr.drop(index=[0, 1, 2, 3])
                                    new_columns = df_lllyr.iloc[0]
                                    df_lllyr = df_lllyr[1:]  # 删除
                                    df_lllyr.columns = new_columns
                                    df_lllyr = df_lllyr[(df_lllyr['二级来源'].isin(['手淘推荐', '手淘搜索'])) & (
                                            df_lllyr['三级来源'] == '汇总')]

                                    # 在最前面增加一列 '流量来源'
                                    df_lllyr.insert(0, '流量来源', df_lllyr['二级来源'])
                                    df_lllyr['一级来源'] = df_lllyr['二级来源']
                                    df_lllyr['二级来源'] = '-'
                                    columns_to_keep = [
                                        '流量来源', '一级来源', '二级来源', '访客数', '下单买家数', '下单转化率',
                                        '浏览量', '店内跳转人数', '跳出本店人数',
                                        '收藏人数', '加购人数', '支付件数', '支付买家数', '支付转化率', '支付金额',
                                        '直接支付买家数', '收藏商品-支付买家数', '粉丝支付买家数',
                                        '加购商品-支付买家数']
                                    # 仅保留指定的字段并按顺序排列
                                    df_lllyr = df_lllyr[columns_to_keep]
                                    df_lllyr.insert(0, '商品图片', '')
                                    df_lllyr.insert(0, '商品ID', product)
                                    df_lllyr.insert(0, '商品名称', '')
                                    df_lllyr.insert(0, '日期', yesterday)
                                    df_lllyr.insert(df_lllyr.columns.get_loc('加购人数') + 1, '总收藏加购人数', 0)
                                    df_lllyr.insert(df_lllyr.columns.get_loc('总收藏加购人数') + 1, '收藏率', '0%')
                                    df_lllyr.insert(df_lllyr.columns.get_loc('收藏率') + 1, '加购率', '0%')
                                    df_lllyr.insert(df_lllyr.columns.get_loc('加购率') + 1, '收藏加购率', '0%')
                                    df_lllyr.insert(df_lllyr.columns.get_loc('支付金额') + 1, '客单价', 0)
                                    df_lllyr.insert(df_lllyr.columns.get_loc('客单价') + 1, 'UV价值', 0)

                                    # page.keyboard.press('Escape')
                                    if not df_lllyr.empty:
                                        threading.Thread(target=auto_data_to_mysql,
                                                         args=(
                                                             df_lllyr, 'ecapp_lllyr', store, dburl, None, None, product,
                                                             today, task_status)).start()
                                        task_status[store][product]['LLLYR'] = True
                                    else:
                                        task_status[store][product]['LLLYR'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                                    pro_flag = True
                                except BaseException as e:
                                    proMsg = '--' + store + '--' + product + '--流量来源-日获取失败:' + str(e)
                                    self.logger.warning(proMsg)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                            # =========================      8.核心概况-实时===不再获取      =========================#
                            if 1 == 2 and not task_status[store][product]['HXGK']:  # 不再获取昨日分时数据
                                try:
                                    hxgk_url = get_setting('TASK_URL')[
                                                   'HXGK'] + today + '%7C' + today + '&dateType=today&itemId='
                                    hxgk_url += product
                                    page.goto(hxgk_url)
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    custom_button_div = page.locator("button:has-text('自定义')").locator('..')
                                    curr_button = custom_button_div.locator("button").locator("span:has-text('实 时')")
                                    if curr_button.count() > 0:
                                        curr_button.click(timeout=3000, force=True)
                                        time.sleep(random.randint(self.min_wait, self.max_wait))

                                    button_selector = 'div#Dts_CcItemArchivesCore button.el-button--success'
                                    max_retries = 3  # 最大重试次数
                                    retries = 0
                                    while retries < max_retries:
                                        try:
                                            # 悬浮弹窗
                                            close_icon = page.query_selector('.anticon-close.close-icon')
                                            if close_icon:
                                                close_icon.click(timeout=3000)
                                            else:
                                                pass
                                        except Exception as e:
                                            pass
                                        try:
                                            page.wait_for_selector(button_selector, timeout=5000)
                                            page.click(button_selector)
                                            break
                                        except:
                                            retries += 1
                                            time.sleep(random.randint(self.min_wait, self.max_wait))
                                            page.goto(hxgk_url)
                                            page.wait_for_load_state()
                                            time.sleep(random.randint(self.min_wait, self.max_wait))
                                        # 使用 JavaScript 点击作为备用
                                    if retries == max_retries:
                                        page.evaluate(f"document.querySelector('{button_selector}').click();")
                                        self.logger.info("核心概况使用了备用点击")
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    max_retries_hxgkr = 3
                                    zcount_hxgkr = 0
                                    while zcount_hxgkr < max_retries_hxgkr:
                                        try:
                                            page.wait_for_selector('.el-dialog__wrapper', timeout=5000)
                                            break
                                        except BaseException as e:
                                            zcount_hxgkr += 1
                                            page.goto(hxgk_url)
                                            page.wait_for_load_state()
                                            page.click(button_selector)
                                            time.sleep(random.randint(self.min_wait, self.max_wait))
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    export_button = page.get_by_role("button", name="导出表格")
                                    export_button_div = export_button.locator('..')
                                    dropdown_button = export_button_div.locator('i.el-icon-arrow-down')
                                    dropdown_button.scroll_into_view_if_needed()
                                    dropdown_button.click()
                                    export_option = page.locator('li:has-text("导出表格 csv(所有列)")')

                                    # 监听下载事件
                                    with page.expect_download() as download_info:
                                        export_option.click()

                                    download = download_info.value
                                    suggested_filename = download.suggested_filename

                                    # 保存下载的文件到指定路径
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_hxgk = self.get_file_to_pd(download_folder, suggested_filename)
                                    # 前两列位置互换
                                    df_hxgk = df_hxgk[
                                        [df_hxgk.columns[1], df_hxgk.columns[0]] + df_hxgk.columns[2:].tolist()]

                                    page.keyboard.press('Escape')
                                    if not df_hxgk.empty:
                                        threading.Thread(target=auto_data_to_mysql,
                                                         args=(df_hxgk, 'ecapp_hxgk', store, dburl, None, None, product,
                                                               today, task_status)).start()
                                        task_status[store][product]['HXGK'] = True
                                    else:
                                        task_status[store][product]['HXGK'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                                    pro_flag = True
                                except BaseException as e:
                                    proMsg = '--' + store + '--' + product + '--核心概况-实时获取失败:' + str(e)
                                    self.logger.warning(proMsg)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                        elif store_type[store] == '京东':  # =====================================================================京东
                            # =========================      6.京东-流量来源表-日      =========================#
                            self.status_signal.emit(f"处理商品流量来源:{curr}/{zCount}")
                            if not task_status[store][product]['LLLYR'] and store == '青岛昌隆文具有限公司':
                                try:
                                    url = get_setting('TASK_URL')['JDLLLYR']
                                    page.goto(url)
                                    # 等待页面加载
                                    page.wait_for_load_state()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    self.remove_jd_ad(page)
                                    #   京东用自营店下载自营和POP的数据,POP的数据计入白雪文具官方旗舰店
                                    store_category_list = ['自营', 'POP']
                                    jd_store = store
                                    if zindex == 0:  # 在第一个产品处额外下载POP的数据
                                        jd_store = '白雪文具官方旗舰店'
                                        for product2 in product_jd_list:
                                            # 1. 根据 ID 定位元素
                                            margin_div = page.locator("#Card1")

                                            # 3. 点经营模式,选自营
                                            target_div = margin_div.locator(
                                                "div:nth-child(3) > div.jmtd-form-item-content")  # 精确选择第三个 div
                                            target_div.click()
                                            target_element = page.locator(
                                                f"//div[@class='jmtd-dropdown-content']//div[@class='jmtd-label' and text()='POP']")
                                            target_element.first.click()

                                            # 日期默认就是昨天,所以不需要设置
                                            # 点击SKU输入框
                                            sku_div = margin_div.locator(
                                                "div:nth-child(7) > div.jmtd-form-item-content")
                                            sku_in_div = sku_div.locator(
                                                "div.jmtd-multiple-base-input-tag-wrapper").first
                                            sku_in_div.click()
                                            # 先删掉之前输入的SKU号
                                            try:
                                                input_element = sku_in_div.locator("input")
                                                input_element.fill("")
                                                sku_close = sku_in_div.locator('span.jmti-close').first
                                                if sku_close.count() > 0:
                                                    sku_close.click()
                                            except:
                                                pass
                                            sku_input = sku_in_div.locator("input")
                                            time.sleep(random.randint(self.min_wait, self.max_wait))
                                            sku_input.type(product2, delay=100)
                                            page.keyboard.press('Enter')

                                            # 4. 点 "确定" 文本的按钮
                                            target_button = margin_div.locator("div.jmtd-form-item-content >> button:has-text('确定')")
                                            target_button.click()  # 点击包含 "确定" 文本的按钮
                                            time.sleep(random.randint(self.min_wait, self.max_wait))

                                            # 找到 id 为 Row1 的 div
                                            content_card_div = page.locator("#Row1")
                                            # 5. 点 "下载数据" 按钮
                                            download_button = content_card_div.locator("button:has-text('下载数据')")
                                            download_button.click()  # 点击下载数据按钮

                                            # 6. 点 "汇总下载"
                                            sum_download_button = page.locator("div.jmtd-dropdown-content:has-text('汇总下载')")
                                            sum_download_button.click()
                                            time.sleep(random.randint(self.min_wait, self.max_wait))

                                            # 获取所有的jmtd-modal元素
                                            # 获取页面上的所有class为'jmtd-label'且文本为'一级渠道'的div元素
                                            channels = page.locator('div.jmtd-label:has-text("一级渠道")')

                                            # 获取第三个'一级渠道' div
                                            third_channel = channels.nth(2)
                                            third_channel.click()
                                            time.sleep(random.randint(1, 2))
                                            page.locator("div.jmtd-label:has-text('二级渠道')").click()
                                            # 获取页面上所有 "确定" 按钮
                                            confirm_buttons = page.locator('button:has-text("确定")')
                                            # 获取最后一个 "确定" 按钮
                                            last_confirm_button = confirm_buttons.nth(confirm_buttons.count() - 1)

                                            # 监听下载事件
                                            with page.expect_download() as download_info:
                                                last_confirm_button.click()

                                            download = download_info.value
                                            suggested_filename = download.suggested_filename

                                            # 7.保存下载的文件到指定路径并读取
                                            download.save_as(os.path.join(download_folder, suggested_filename))
                                            df_jdlllyr = self.get_file_to_pd(download_folder, suggested_filename)
                                            df_jdlllyr = df_jdlllyr.copy()
                                            # 只保留搜索和推荐
                                            df_jdlllyr = df_jdlllyr[df_jdlllyr['二级渠道'].isin(['搜索', '推荐'])]
                                            df_jdlllyr['日期'] = yesterday0
                                            # 日期后增加列
                                            df_jdlllyr.insert(df_jdlllyr.columns.get_loc('日期') + 1, '商品id',
                                                              product2)
                                            df_jdlllyr.insert(df_jdlllyr.columns.get_loc('商品id') + 1, '流量来源',
                                                              '京东平台')

                                            # 删除实际存在的列
                                            columns_to_drop = ['人均浏览量(人均PV)', '平均停留时长(秒)', '关注客户数',
                                                               '加购转换率',
                                                               '下单转换率', '下单金额', '成交子单数']
                                            existing_columns_to_drop = list(
                                                set(columns_to_drop) & set(df_jdlllyr.columns))  # 获取存在的列
                                            df_jdlllyr = df_jdlllyr.drop(existing_columns_to_drop, axis=1)

                                            task_status[jd_store][product2]['HXGK'] = True  # 不需要获取,直接设为True
                                            if not df_jdlllyr.empty:
                                                threading.Thread(target=auto_data_to_mysql, args=(
                                                    df_jdlllyr, 'ecapp_lllyr', jd_store, dburl, None, '京东', product2,
                                                    today, task_status)).start()
                                                task_status[jd_store][product2]['LLLYR'] = True
                                            else:
                                                task_status[jd_store][product2]['LLLYR'] = True
                                                threading.Thread(target=only_up_task_status,
                                                                 args=(dburl, today, task_status)).start()

                                    margin_div = page.locator("#Card1")
                                    # 3. 点经营模式,选自营
                                    target_div = margin_div.locator(
                                        "div:nth-child(3) > div.jmtd-form-item-content")  # 精确选择第三个 div
                                    target_div.click()
                                    target_element = page.locator(
                                        f"//div[@class='jmtd-dropdown-content']//div[@class='jmtd-label' and text()='自营']")
                                    target_element.first.click()

                                    # 日期默认就是昨天,所以不需要设置
                                    # 点击SKU输入框
                                    sku_div = margin_div.locator("div:nth-child(7) > div.jmtd-form-item-content")
                                    sku_in_div = sku_div.locator("div.jmtd-multiple-base-input-tag-wrapper").first
                                    sku_in_div.click()
                                    # 先删掉之前输入的SKU号
                                    try:
                                        input_element = sku_in_div.locator("input")
                                        input_element.fill("")
                                        sku_close = sku_in_div.locator('span.jmti-close').first
                                        if sku_close.count() > 0:
                                            sku_close.click()
                                    except:
                                        pass
                                    sku_input = sku_in_div.locator("input")
                                    sku_input.type(product, delay=100)
                                    page.keyboard.press('Enter')
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 4. 点 "确定" 文本的按钮
                                    target_button = margin_div.locator(
                                        "div.jmtd-form-item-content >> button:has-text('确定')")
                                    target_button.click()  # 点击包含 "确定" 文本的按钮
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 找到 id 为 Row1 的 div
                                    content_card_div = page.locator("#Row1")
                                    # 5. 点 "下载数据" 按钮
                                    download_button = content_card_div.locator("button:has-text('下载数据')")
                                    download_button.click()  # 点击下载数据按钮
                                    # 6. 点 "汇总下载"
                                    sum_download_button = page.locator("div.jmtd-dropdown-content:has-text('汇总下载')")
                                    sum_download_button.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))

                                    # 获取所有的jmtd-modal元素
                                    # 获取页面上的所有class为'jmtd-label'且文本为'一级渠道'的div元素
                                    channels = page.locator('div.jmtd-label:has-text("一级渠道")')

                                    # 获取第三个'一级渠道' div
                                    third_channel = channels.nth(2)
                                    third_channel.click()
                                    time.sleep(random.randint(1, 2))
                                    page.locator("div.jmtd-label:has-text('二级渠道')").click()
                                    # 获取页面上所有 "确定" 按钮
                                    confirm_buttons = page.locator('button:has-text("确定")')
                                    # 获取最后一个 "确定" 按钮
                                    last_confirm_button = confirm_buttons.nth(confirm_buttons.count() - 1)

                                    # 监听下载事件
                                    with page.expect_download() as download_info:
                                        last_confirm_button.click()

                                    download = download_info.value
                                    suggested_filename = download.suggested_filename

                                    # 7.保存下载的文件到指定路径并读取
                                    download.save_as(os.path.join(download_folder, suggested_filename))
                                    df_jdlllyr = self.get_file_to_pd(download_folder, suggested_filename)
                                    df_jdlllyr = df_jdlllyr.copy()
                                    # 只保留搜索和推荐的
                                    # df_jdlllyr = df_jdlllyr[df_jdlllyr['二级渠道'] == '搜索']
                                    df_jdlllyr = df_jdlllyr[df_jdlllyr['二级渠道'].isin(['搜索', '推荐'])]
                                    df_jdlllyr['日期'] = yesterday0
                                    # 日期后增加列
                                    df_jdlllyr.insert(df_jdlllyr.columns.get_loc('日期') + 1, '商品id', product)
                                    df_jdlllyr.insert(df_jdlllyr.columns.get_loc('商品id') + 1, '流量来源', '京东平台')

                                    # df_jdlllyr = df_jdlllyr.drop(['人均浏览量(人均PV)', '平均停留时长(秒)','关注客户数','加购转换率','下单转换率','下单金额', '成交子单数'],axis=1)
                                    # 删除实际存在的列
                                    columns_to_drop = ['人均浏览量(人均PV)', '平均停留时长(秒)', '关注客户数',
                                                       '加购转换率', '下单转换率', '下单金额', '成交子单数']
                                    existing_columns_to_drop = list(
                                        set(columns_to_drop) & set(df_jdlllyr.columns))  # 获取存在的列
                                    df_jdlllyr = df_jdlllyr.drop(existing_columns_to_drop, axis=1)

                                    # task_status[store][product]['HXGK'] = True#不需要获取,直接设为True
                                    if not df_jdlllyr.empty:
                                        threading.Thread(target=auto_data_to_mysql, args=(
                                            df_jdlllyr, 'ecapp_lllyr', store, dburl, None, store_type[store], product,
                                            today, task_status)).start()
                                        task_status[store][product]['LLLYR'] = True
                                    else:
                                        task_status[store][product]['LLLYR'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                                    pro_flag = True
                                except BaseException as e:
                                    proMsg = '--' + store + '--' + product + '--流量来源-日获取失败:' + str(e)
                                    self.logger.warning(proMsg)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                        elif store_type[store] == '拼多多':
                            # =========================      4.核心概况-日      =========================#
                            self.status_signal.emit(f"处理商品核心概况:{curr}/{zCount}")
                            if not task_status[store][product]['HXGKR']:
                                try:
                                    pddhxgkr_url = get_setting('TASK_URL')['PDDHXGKR']
                                    if page.url != pddhxgkr_url:
                                        page.goto(pddhxgkr_url)
                                        page.wait_for_load_state()
                                        time.sleep(random.randint(self.min_wait, self.max_wait))
                                    try:
                                        # 等待包含 "新消息" 的 div 出现,点击关闭
                                        page.wait_for_selector("div.ImportantList_msgbox-header__1hGG1",
                                                               state="attached", timeout=3000)
                                        close_icon = page.query_selector(
                                            "div.ImportantList_msgbox-header__1hGG1 > div > i")
                                        if close_icon:
                                            close_icon.click(timeout=3000)
                                    except Exception as e:
                                        pass
                                    input_container = page.query_selector("div[data-testid='beast-core-input']")
                                    input_container.scroll_into_view_if_needed()
                                    # 定位到第二个 beast-core-radioGroup
                                    radio_group = page.query_selector(
                                        "div[data-testid='beast-core-radioGroup']:nth-match(div[data-testid='beast-core-radioGroup'], 2)")
                                    yesterday_option = radio_group.query_selector("label:has-text('昨日')")
                                    yesterday_option.click(force=True)
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 在该元素内部找到 input 元素
                                    input_element = input_container.query_selector("input")
                                    # 清空输入框中的值
                                    input_element.fill("")
                                    # 模拟人类输入行为，逐字符输入新值
                                    input_element.type(product, delay=100)

                                    # 定位到 data-testid="beast-core-input-suffix" 的 div 元素并点击
                                    suffix_element = input_container.query_selector(
                                        "div[data-testid='beast-core-input-suffix']")
                                    suffix_element.click()
                                    time.sleep(random.randint(self.min_wait, self.max_wait))
                                    # 移除“数据详情”和右侧悬浮按钮的遮挡元素
                                    page.evaluate("""
                                                () => {
                                                    const element = document.getElementById('umd_kits_home_entry');
                                                        if (element) {
                                                            element.remove();
                                                        }
                                                    const elements = document.querySelectorAll('td');
                                                    elements.forEach(el => {
                                                        if (el.textContent.includes('数据详情')) {
                                                            el.remove();
                                                        }
                                                    });
                                                }
                                            """)
                                    tr_element = page.locator("tr[data-testid='beast-core-table-body-tr']")
                                    # 获取 tr 内除第一个 td 外的所有 td 元素
                                    td_elements = tr_element.locator("td:not(:first-child)").all()
                                    name_list = ['商品访客数', '商品浏览量', '销售额', '成交件数', '成交订单数',
                                                 '成交买家数', '成交转化率']
                                    hxgkr_dict = {'商品访客数': 0, '商品浏览量': 0, '销售额': 0, '成交件数': 0,
                                                  '成交订单数': 0, '成交买家数': 0, '成交转化率': '0%'}
                                    for index, td in enumerate(td_elements, 0):
                                        if index > 6:
                                            break
                                        safe_name = name_list[index]
                                        try:
                                            target_element = td.locator("div > div > span").first
                                            # 确保目标元素可见
                                            target_element.scroll_into_view_if_needed()
                                            # 截图并保存
                                            screen_path = f"{img_path}/{safe_name}.png"
                                            target_element.screenshot(path=screen_path, timeout=5000,
                                                                      animations="disabled", caret="initial")
                                            hxgkr_dict[safe_name] = self.ocr_img_num(screen_path, exe_path)
                                        except Exception as e:
                                            self.logger.error(f"拼多多数据总览图像识别失败{str(e)}")
                                    df = DataFrame([hxgkr_dict])
                                    df = df.apply(to_numeric, errors='ignore')
                                    verbose_names = [
                                        '商品名称', '商品ID', '商品图片', '类别', '日期', '商品访客数',
                                        '商品微详情访客数', '商品浏览量', '平均停留时长', '跳失率',
                                        '加购件数', '加购人数', '收藏人数', '访问加购转化率', '访问收藏转化率',
                                        '下单买家数', '成交订单数', '下单金额',
                                        '下单转化率', '成交买家数', '成交件数', '销售额', '成交转化率',
                                        '聚划算支付金额', '支付新买家数', '支付老买家数',
                                        '老买家支付金额', 'UV价值', '人均浏览量', '详情页跳出人数',
                                        '聚划算支付金额占比', '件单价', '客单价', '成功退款金额'
                                    ]
                                    pddhxgkr_df = DataFrame(columns=verbose_names)
                                    pddhxgkr_df[name_list] = df[name_list]
                                    pddhxgkr_df['商品ID'] = product
                                    pddhxgkr_df['类别'] = '本品'
                                    pddhxgkr_df['日期'] = yesterday0
                                    if not pddhxgkr_df.empty:
                                        threading.Thread(target=auto_data_to_mysql,
                                                         args=(
                                                             pddhxgkr_df, 'ecapp_hxgkr', store, dburl, None, None, product,
                                                             today, task_status)).start()
                                        task_status[store][product]['HXGKR'] = True
                                        task_status[store][product]['LLLYR'] = True
                                    else:
                                        task_status[store][product]['HXGKR'] = True
                                        task_status[store][product]['LLLYR'] = True
                                        threading.Thread(target=only_up_task_status,
                                                         args=(dburl, today, task_status)).start()
                                    pro_flag = True
                                except BaseException as e:
                                    proMsg = '--' + store + '--' + product + '--核心概况-日获取失败:' + str(e)
                                    self.logger.warning(proMsg)
                                time.sleep(random.randint(self.min_wait, self.max_wait))
                        if pro_flag:
                            time.sleep(random.randint(self.min_wait, self.max_wait))
                        else:
                            time.sleep(1)
                    # ===========================   下载过任何数据时,关闭浏览器,并增加等待时间
                    # 每运行完一个店铺,关闭浏览器重开一个新的,降低被检测概率
                    try:
                        context.close()
                    except:
                        pass
                    try:
                        browser.close()
                    except:
                        pass
                    check_res_task = self.check_all_task(task_status)
                    proMsg = '--' + store + '本次运行完成--当前状态:[' + check_res_task[store][
                        'store_false_count'] + ']个店铺级数据表未获取完成' + ',[' + check_res_task[store][
                                 'product_false_count'] + ']个产品未获取完成'
                    self.logger.info(proMsg)
                    waitTime = random.randint(20, 30)
                    proMsg = '--' + str(waitTime) + '秒后,将进行下一步'
                    self.logger.info(proMsg)
                    self.status_signal.emit(proMsg)
                    time.sleep(waitTime)
                else:
                    proMsg = '--' + store + '--数据已全部获取完毕,跳过'
                    self.logger.info(proMsg)
        except BaseException as e:
            proMsg = '本次任务-出现错误:' + str(e)
            self.logger.error(proMsg)
        finally:
            try:
                context.close()
            except:
                pass
            try:
                browser.close()
            except:
                pass
        self.logger.info('本次线程结束')
        self.status_signal.emit("准备就绪")
        self.finished.emit()
